From ee59eaed33d590fd898781974e26ccb77a14bc4e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 1 Apr 2025 15:28:00 +0200 Subject: [PATCH 01/51] wip --- attachments/README.md | 0 attachments/build.gradle.kts | 151 ++++++++ attachments/gradle.properties | 0 .../AbstractLocalStorageAdapter.android.kt | 8 + .../powersync/attachments/AttachmentsTest.kt | 50 +++ .../attachments/testutils/TestUtils.kt | 7 + .../attachments/testutils/UserRow.kt | 35 ++ .../AbstractAttachmentQueue.kt | 359 ++++++++++++++++++ .../com.powersync.attachments/Attachment.kt | 42 ++ .../AttachmentService.kt | 176 +++++++++ .../LocalStorageAdapter.kt | 29 ++ .../RemoteStorageAdapter.kt | 25 ++ .../SyncErrorHandler.kt | 32 ++ .../storage/AbstractLocalStorageAdapter.kt | 7 + .../storage/IOLocalStorageAdapter.kt | 67 ++++ .../sync/SyncingService.kt | 231 +++++++++++ .../AbstractLocalStorageAdapter.ios.kt | 8 + .../testutils/TestUtils.iosSimulatorArm64.kt | 15 + .../AbstractLocalStorageAdapter.jvm.kt | 7 + .../attachments/testutils/TestUtils.jvm.kt | 11 + gradle/libs.versions.toml | 4 + plugins/build-plugin/build.gradle.kts | 16 + plugins/build-plugin/settings.gradle.kts | 1 + .../src/main/kotlin/SharedBuildPlugin.kt | 74 ++++ plugins/settings.gradle.kts | 3 +- settings.gradle.kts | 1 + 26 files changed, 1358 insertions(+), 1 deletion(-) create mode 100644 attachments/README.md create mode 100644 attachments/build.gradle.kts create mode 100644 attachments/gradle.properties create mode 100644 attachments/src/androidMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.android.kt create mode 100644 attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt create mode 100644 attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestUtils.kt create mode 100644 attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/UserRow.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/SyncErrorHandler.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt create mode 100644 attachments/src/iosMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.ios.kt create mode 100644 attachments/src/iosSimulatorArm64Test/kotlin/com/powersync/attachments/testutils/TestUtils.iosSimulatorArm64.kt create mode 100644 attachments/src/jvmMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.jvm.kt create mode 100644 attachments/src/jvmTest/kotlin/com/powersync/attachments/testutils/TestUtils.jvm.kt create mode 100644 plugins/build-plugin/build.gradle.kts create mode 100644 plugins/build-plugin/settings.gradle.kts create mode 100644 plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt diff --git a/attachments/README.md b/attachments/README.md new file mode 100644 index 00000000..e69de29b diff --git a/attachments/build.gradle.kts b/attachments/build.gradle.kts new file mode 100644 index 00000000..a18df883 --- /dev/null +++ b/attachments/build.gradle.kts @@ -0,0 +1,151 @@ +import com.powersync.plugins.sonatype.setupGithubRepository +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinTest + + +plugins { + alias(libs.plugins.kotlinMultiplatform) + alias(libs.plugins.kotlinSerialization) + alias(libs.plugins.androidLibrary) + alias(libs.plugins.mavenPublishPlugin) + alias(libs.plugins.kotlinter) + id("com.powersync.plugins.sonatype") + alias(libs.plugins.mokkery) + alias(libs.plugins.kotlin.atomicfu) + id("com.powersync.plugins.sharedbuild") +} + +kotlin { + androidTarget { + publishLibraryVariants("release", "debug") + publishLibraryVariants("release", "debug") + + @OptIn(ExperimentalKotlinGradlePluginApi::class) + compilerOptions { + jvmTarget.set(JvmTarget.JVM_17) + } + } + jvm { + @OptIn(ExperimentalKotlinGradlePluginApi::class) + compilerOptions { + jvmTarget.set(JvmTarget.JVM_1_8) + // https://jakewharton.com/kotlins-jdk-release-compatibility-flag/ + freeCompilerArgs.add("-Xjdk-release=8") + } + } + + iosX64() + iosArm64() + iosSimulatorArm64() + + targets.withType { + compilations.named("main") { + compileTaskProvider { + compilerOptions.freeCompilerArgs.add("-Xexport-kdoc") + } + } + } + + explicitApi() + + applyDefaultHierarchyTemplate() + sourceSets { + all { + languageSettings { + optIn("kotlinx.cinterop.ExperimentalForeignApi") + } + } + + val commonIntegrationTest by creating { + dependsOn(commonTest.get()) + } + + commonMain.dependencies { + api(project(":core")) + implementation(libs.uuid) + implementation(libs.kotlin.stdlib) + implementation(libs.ktor.client.core) + implementation(libs.ktor.client.contentnegotiation) + implementation(libs.ktor.serialization.json) + implementation(libs.kotlinx.io) + implementation(libs.kotlinx.coroutines.core) + implementation(libs.kotlinx.datetime) + implementation(libs.stately.concurrency) + implementation(libs.configuration.annotations) + api(projects.persistence) + api(libs.kermit) + } + + androidMain { + dependencies.implementation(libs.ktor.client.okhttp) + } + + jvmMain { + dependencies { + implementation(libs.ktor.client.okhttp) + } + } + + iosMain.dependencies { + implementation(libs.ktor.client.ios) + } + + commonTest.dependencies { + implementation(libs.kotlin.test) + implementation(libs.test.coroutines) + implementation(libs.test.turbine) + implementation(libs.kermit.test) + implementation(libs.ktor.client.mock) + implementation(libs.test.turbine) + } + + jvmTest.get().dependsOn(commonIntegrationTest) + iosSimulatorArm64Test.orNull?.dependsOn(commonIntegrationTest) + } +} + +android { + compileOptions { + targetCompatibility = JavaVersion.VERSION_17 + } + + buildFeatures { + buildConfig = true + } + + buildTypes { + release { + buildConfigField("boolean", "DEBUG", "false") + } + debug { + buildConfigField("boolean", "DEBUG", "true") + } + } + + namespace = "com.powersync.attachments" + compileSdk = + libs.versions.android.compileSdk + .get() + .toInt() + defaultConfig { + minSdk = + libs.versions.android.minSdk + .get() + .toInt() + consumerProguardFiles("proguard-rules.pro") + } +} + +tasks.withType { + testLogging { + events("PASSED", "FAILED", "SKIPPED") + exceptionFormat = TestExceptionFormat.FULL + showStandardStreams = true + showStackTraces = true + } +} + +setupGithubRepository() diff --git a/attachments/gradle.properties b/attachments/gradle.properties new file mode 100644 index 00000000..e69de29b diff --git a/attachments/src/androidMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.android.kt b/attachments/src/androidMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.android.kt new file mode 100644 index 00000000..af7f8b76 --- /dev/null +++ b/attachments/src/androidMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.android.kt @@ -0,0 +1,8 @@ +package com.powersync.attachments.storage + +import android.os.Environment +import com.powersync.attachments.LocalStorageAdapter + +public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { + actual override fun getUserStorageDirectory(): String = Environment.getDataDirectory().absolutePath +} diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt new file mode 100644 index 00000000..899ac86b --- /dev/null +++ b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt @@ -0,0 +1,50 @@ +package com.powersync.attachments + +import co.touchlab.kermit.ExperimentalKermitApi +import com.powersync.PowerSyncDatabase +import com.powersync.attachments.testutils.UserRow +import com.powersync.db.schema.Schema +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test + +@OptIn(ExperimentalKermitApi::class) +class AttachmentsTest { + private lateinit var database: PowerSyncDatabase + + private fun openDB() = + PowerSyncDatabase( + factory = com.powersync.attachments.testutils.factory, + schema = Schema(UserRow.table), + dbFilename = "testdb", + ) + + @BeforeTest + fun setupDatabase() { + database = openDB() + + runBlocking { + database.disconnectAndClear(true) + } + } + + @AfterTest + fun tearDown() { + runBlocking { + if (!database.closed) { + database.disconnectAndClear(true) + database.close() + } + } + com.powersync.attachments.testutils + .cleanup("testdb") + } + + @Test + fun testLinksPowerSync() = + runTest { + database.get("SELECT powersync_rs_version();") { it.getString(0)!! } + } +} diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestUtils.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestUtils.kt new file mode 100644 index 00000000..4684097c --- /dev/null +++ b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestUtils.kt @@ -0,0 +1,7 @@ +package com.powersync.attachments.testutils + +import com.powersync.DatabaseDriverFactory + +expect val factory: DatabaseDriverFactory + +expect fun cleanup(path: String) diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/UserRow.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/UserRow.kt new file mode 100644 index 00000000..63d550b5 --- /dev/null +++ b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/UserRow.kt @@ -0,0 +1,35 @@ +package com.powersync.attachments.testutils + +import com.powersync.db.SqlCursor +import com.powersync.db.getString +import com.powersync.db.getStringOptional +import com.powersync.db.schema.Column +import com.powersync.db.schema.Table + +data class UserRow( + val id: String, + val name: String, + val email: String, + val photoId: String?, +) { + companion object { + fun from(cursor: SqlCursor): UserRow = + UserRow( + id = cursor.getString("id"), + name = cursor.getString("name"), + email = cursor.getString("email"), + photoId = cursor.getStringOptional("photo_id"), + ) + + val table = + Table( + name = "users", + columns = + listOf( + Column.text("name"), + Column.text("email"), + Column.text("photo_id"), + ), + ) + } +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt new file mode 100644 index 00000000..a4ccdef6 --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt @@ -0,0 +1,359 @@ +package com.powersync.attachments + +import co.touchlab.kermit.Logger +import com.powersync.PowerSyncDatabase +import com.powersync.PowerSyncException +import com.powersync.attachments.sync.SyncingService +import com.powersync.db.runWrappedSuspending +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.io.files.Path +import kotlin.coroutines.cancellation.CancellationException +import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes + +/** + * A watched attachment record item. + * This is usually returned from watching all relevant attachment IDs. + */ +public data class WatchedAttachmentItem( + /** + * Id for the attachment record + */ + public val id: String, + /** + * File extension used to determine an internal filename for storage if no [filename] is provided + */ + public val fileExtension: String? = null, + /** + * Filename to store the attachment with + */ + public val filename: String? = null, +) { + init { + require(fileExtension != null || filename != null) { + "Either fileExtension or filename must be provided." + } + } +} + +/** + * Abstract class used to implement the attachment queue + * Requires a PowerSyncDatabase, an implementation of + * AbstractRemoteStorageAdapter and an attachment directory name which will + * determine which folder attachments are stored into. + */ +public abstract class AbstractAttachmentQueue( + /** + * PowerSync database client + */ + private val db: PowerSyncDatabase, + /** + * Adapter which interfaces with the remote storage backend + */ + private val remoteStorage: RemoteStorageAdapter, + /** + * Provides access to local filesystem storage methods + */ + private val localStorage: LocalStorageAdapter = IOLocalStorageAdapter(), + /** + * Directory where attachment files will be written to disk + */ + private val attachmentDirectoryName: String = DEFAULT_ATTACHMENTS_DIRECTORY_NAME, + /** + * SQLite table where attachment state will be recorded + */ + private val attachmentsQueueTableName: String = DEFAULT_TABLE_NAME, + /** + * Attachment operation error handler. This specified if failed attachment operations + * should be retried. + */ + private val errorHandler: SyncErrorHandler?, + /** + * Periodic interval to trigger attachment sync operations + */ + private val syncInterval: Duration = 5.minutes, + /** + * Creates a list of subdirectories in the {attachmentDirectoryName} directory + */ + private val subdirectories: List? = null, + /** + * Logging interface used for all log operations + */ + private val logger: Logger = Logger, +) { + public companion object { + public const val DEFAULT_TABLE_NAME: String = "attachments" + public const val DEFAULT_ATTACHMENTS_DIRECTORY_NAME: String = "attachments" + } + + /** + * Service which provides access to attachment records. + * Use this to: + * - Query all current attachment records + * - Create new attachment records for upload/download + */ + public val attachmentsService: AttachmentService = + AttachmentService(db, attachmentsQueueTableName, logger) + + /** + * Syncing service for this attachment queue. + * This processes attachment records and performs relevant upload, download and delete + * operations. + */ + private val syncingService: SyncingService = + SyncingService( + remoteStorage, + localStorage, + attachmentsService, + ::getLocalUri, + errorHandler, + logger, + ) + + private var syncStatusJob: Job? = null + private val mutex = Mutex() + + public var closed: Boolean = false + + /** + * Initialize the attachment queue by + * 1. Creating attachments directory + * 2. Adding watches for uploads, downloads, and deletes + * 3. Adding trigger to run uploads, downloads, and deletes when device is online after being offline + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun start(): Unit = + runWrappedSuspending { + mutex.withLock { + if (closed) { + throw Exception("Attachment queue has been closed") + } + // Ensure the directory where attachments are downloaded, exists + localStorage.makeDir(getStorageDirectory()) + + subdirectories?.forEach { subdirectory -> + localStorage.makeDir(Path(getStorageDirectory(), subdirectory).toString()) + } + + // Start watching for changes + val scope = CoroutineScope(Dispatchers.IO) + + syncingService.startPeriodicSync(syncInterval) + + // Listen for connectivity changes + syncStatusJob = + scope.launch { + scope.launch { + db.currentStatus.asFlow().collect { status -> + if (status.connected) { + syncingService.triggerSync() + } + } + } + + scope.launch { + // Watch local attachment relationships and sync the attachment records + watchAttachments().collect { items -> + processWatchedAttachments(items) + } + } + } + } + } + + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun close(): Unit = + runWrappedSuspending { + mutex.withLock { + if (closed) { + return@runWrappedSuspending + } + + syncStatusJob?.cancel() + syncStatusJob?.join() + syncingService.close() + + closed = true + } + } + + /** + * Creates a watcher for the current state of local attachments + * ```kotlin + * public fun watchAttachments(): Flow> = + * db.watch( + * sql = + * """ + * SELECT + * photo_id as id + * FROM + * checklists + * WHERE + * photo_id IS NOT NULL + * """, + * ) { cursor -> + * WatchedAttachmentItem( + * id = cursor.getString("id"), + * fileExtension = "jpg", + * ) + * } + * ``` + */ + @Throws(PowerSyncException::class) + public abstract fun watchAttachments(): Flow> + + /** + * Resolves the filename for new attachment items. + * A new attachment from [watchAttachments] might not include a filename. + * Concatenates the attachment ID an extension by default. + * This method can be overriden for custom behaviour. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun resolveNewAttachmentFilename( + attachmentId: String, + fileExtension: String?, + ): String = "$attachmentId.$fileExtension" + + /** + * Processes attachment items returned from [watchAttachments]. + * The default implementation assets the items returned from [watchAttachments] as the definitive + * state for local attachments. + * + * Records currently in the attachment queue which are not present in the items are deleted from + * the queue. + * + * Received items which are not currently in the attachment queue are assumed scheduled for + * download. This requires that locally created attachments should be created with [saveFile] + * before assigning the attachment ID to the relevant watched tables. + * + * This method can be overriden for custom behaviour. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun processWatchedAttachments(items: List): Unit = + runWrappedSuspending { + val currentAttachments = attachmentsService.getAttachments() + val attachmentUpdates = mutableListOf() + + for (item in items) { + val existingQueueItem = currentAttachments.find { it.id == item.id } + + if (existingQueueItem == null) { + // This item should be added to the queue + // This item is assumed to be coming from an upstream sync + // Locally created new items should be persisted using [saveFile] before + // this point. + val filename = + resolveNewAttachmentFilename( + attachmentId = item.id, + fileExtension = item.fileExtension, + ) + + attachmentUpdates.add( + Attachment( + id = item.id, + filename = filename, + state = AttachmentState.QUEUED_DOWNLOAD.ordinal, + ), + ) + } + } + + // Remove any items not specified in the items + currentAttachments + .filter { null == items.find { update -> update.id == it.id } } + .forEach { + attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED.ordinal)) + } + } + + /** + * A function which creates a new attachment locally. This new attachment is queued for upload + * after creation. + * The relevant attachment file should be persisted to disk before calling this method. + * The default implementation assumes the attachment file has been written to the path + * ```kotlin + * val path = getLocalFilePathSuffix( + * resolveNewAttachmentFilename( + * attachmentId = attachmentId, + * fileExtension = fileExtension, + * )) + * ) + * ``` + * This method can be overriden for custom behaviour. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun saveFile( + attachmentId: String, + size: Long, + mediaType: String, + fileExtension: String?, + ): Attachment = + runWrappedSuspending { + val filename = + resolveNewAttachmentFilename( + attachmentId = attachmentId, + fileExtension = fileExtension, + ) + + return@runWrappedSuspending attachmentsService.saveAttachment( + Attachment( + id = attachmentId, + filename = filename, + size = size, + mediaType = mediaType, + state = AttachmentState.QUEUED_UPLOAD.ordinal, + localUri = getLocalFilePathSuffix(filename).toString(), + ), + ) + } + + /** + * A function which creates an attachment delete operation locally. This operation is queued + * for delete after creating. + * The default implementation assumes the attachment record already exists locally. An exception + * is thrown if the record does not exist locally. + * This method can be overriden for custom behaviour. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun deleteFile(attachmentId: String): Attachment = + runWrappedSuspending { + val attachment = + attachmentsService.getAttachment(attachmentId) + ?: throw Exception("Attachment record with id $attachmentId was not found.") + + return@runWrappedSuspending attachmentsService.saveAttachment( + attachment.copy(state = AttachmentState.QUEUED_DELETE.ordinal), + ) + } + + /** + * Returns the local file path for the given filename, used to store in the database. + * Example: filename: "attachment-1.jpg" returns "attachments/attachment-1.jpg" + */ + public fun getLocalFilePathSuffix(filename: String): String = Path(attachmentDirectoryName, filename).toString() + + /** + * Returns the directory where attachments are stored on the device, used to make dir + * Example: "/data/user/0/com.yourdomain.app/files/attachments/" + */ + public fun getStorageDirectory(): String { + val userStorageDirectory = localStorage.getUserStorageDirectory() + return Path(userStorageDirectory, attachmentDirectoryName).toString() + } + + /** + * Return users storage directory with the attachmentPath use to load the file. + * Example: filePath: "attachments/attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg" + */ + public fun getLocalUri(filePath: String): String { + val storageDirectory = getStorageDirectory() + return Path(storageDirectory, filePath).toString() + } +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt new file mode 100644 index 00000000..548e8d9e --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt @@ -0,0 +1,42 @@ +package com.powersync.attachments + +import com.powersync.db.SqlCursor +import com.powersync.db.getLong +import com.powersync.db.getString + +/** + * Enum for the attachment state + */ +public enum class AttachmentState { + QUEUED_DOWNLOAD, + QUEUED_UPLOAD, + QUEUED_DELETE, + SYNCED, + ARCHIVED, +} + +/** + * Data class representing an attachment + */ +public data class Attachment( + val id: String, + val timestamp: Long = 0, + val filename: String, + val state: Int = AttachmentState.QUEUED_DOWNLOAD.ordinal, + val localUri: String? = null, + val mediaType: String? = null, + val size: Long? = null, +) { + public companion object { + public fun fromCursor(cursor: SqlCursor): Attachment = + Attachment( + id = cursor.getString(name = "id"), + timestamp = cursor.getLong("timestamp"), + filename = cursor.getString(name = "filename"), + localUri = cursor.getString(name = "local_uri"), + mediaType = cursor.getString(name = "media_type"), + size = cursor.getLong("size"), + state = cursor.getLong("state").toInt(), + ) + } +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt new file mode 100644 index 00000000..9f82f4d8 --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt @@ -0,0 +1,176 @@ +package com.powersync.attachments + +import co.touchlab.kermit.Logger +import com.powersync.PowerSyncDatabase +import com.powersync.db.internal.ConnectionContext +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.map +import kotlinx.datetime.Clock + +/** + * Service for interacting with the local attachment records. + */ +public class AttachmentService( + private val db: PowerSyncDatabase, + private val tableName: String, + private val logger: Logger, +) { + /** + * Table used for storing attachments in the attachment queue. + */ + private val table: String + get() = tableName + + /** + * Delete the attachment from the attachment queue. + */ + public suspend fun deleteAttachment(id: String) { + db.execute("DELETE FROM $table WHERE id = ?", listOf(id)) + } + + /** + * Set the state of the attachment to ignore. + */ + public suspend fun ignoreAttachment(id: String) { + db.execute( + "UPDATE $table SET state = ${AttachmentState.ARCHIVED.ordinal} WHERE id = ?", + listOf(id), + ) + } + + /** + * Get the attachment from the attachment queue using an ID. + */ + public suspend fun getAttachment(id: String): Attachment? = + db.getOptional("SELECT * FROM $table WHERE id = ?", listOf(id)) { + Attachment.fromCursor(it) + } + + /** + * Save the attachment to the attachment queue. + */ + public suspend fun saveAttachment(attachment: Attachment): Attachment = + db.writeLock { ctx -> + upsertAttachment(attachment, ctx) + } + + /** + * Save the attachments to the attachment queue. + */ + public suspend fun saveAttachments(attachments: List) { + if (attachments.isEmpty()) { + return + } + + db.writeTransaction { tx -> + for (attachment in attachments) { + upsertAttachment(attachment, tx) + } + } + } + + /** + * Get all the ID's of attachments in the attachment queue. + */ + public suspend fun getAttachmentIds(): List = + db.getAll( + "SELECT id FROM $table WHERE id IS NOT NULL", + ) { it.getString(0)!! } + + public suspend fun getAttachments(): List = + db.getAll( + "SELECT * FROM $table WHERE id IS NOT NULL", + ) { Attachment.fromCursor(it) } + + /** + * Gets all the active attachments which require an operation to be performed. + */ + public suspend fun getActiveAttachments(): List = + db.getAll( + """ + SELECT + * + FROM + $table + WHERE + state = ${AttachmentState.QUEUED_DOWNLOAD.ordinal} + OR state = ${AttachmentState.QUEUED_DELETE.ordinal} + OR state = ${AttachmentState.QUEUED_UPLOAD.ordinal} + """, + listOf(AttachmentState.ARCHIVED.ordinal), + ) { Attachment.fromCursor(it) } + + /** + * Watcher for changes to attachments table. + * Once a change is detected it will initiate a sync of the attachments + */ + public fun watchActiveAttachments(): Flow { + logger.i("Watching attachments...") + return db + .watch( + """ + SELECT + id + FROM + $table + WHERE + state = ${AttachmentState.QUEUED_DOWNLOAD.ordinal} + OR state = ${AttachmentState.QUEUED_DELETE.ordinal} + OR state = ${AttachmentState.QUEUED_UPLOAD.ordinal} + """, + ) { } + // We only use changes here to trigger a sync consolidation + .map { Unit } + } + + /** + * Helper function to clear the attachment queue + * Currently only used for testing purposes. + */ + public suspend fun clearQueue() { + logger.i("Clearing attachment queue...") + db.execute("DELETE FROM $table") + } + + /** + * Delete attachments which have been archived + */ + public suspend fun deleteArchivedAttachments() { + db.execute( + """ + DELETE FROM $table + WHERE state = ${AttachmentState.ARCHIVED.ordinal} + """, + ) + } + + private fun upsertAttachment( + attachment: Attachment, + context: ConnectionContext, + ): Attachment { + val updatedRecord = + attachment.copy( + timestamp = Clock.System.now().toEpochMilliseconds(), + ) + + context.execute( + """ + INSERT OR REPLACE INTO + $table (id, timestamp, filename, local_uri, media_type, size, state) + VALUES + (?, ?, ?, ?, ?, ?, ?) + """, + listOf( + updatedRecord.id, + updatedRecord.timestamp, + updatedRecord.filename, + updatedRecord.localUri, + updatedRecord.mediaType, + updatedRecord.size, + updatedRecord.state, + ), + ) + + return attachment + } +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt new file mode 100644 index 00000000..ad830c8e --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt @@ -0,0 +1,29 @@ +package com.powersync.attachments + +/** + * Storage adapter for local storage + */ +public interface LocalStorageAdapter { + public suspend fun saveFile( + filePath: String, + data: ByteArray, + ): Unit + + public suspend fun readFile( + filePath: String, + mediaType: String? = null, + ): ByteArray + + public suspend fun deleteFile(filePath: String): Unit + + public suspend fun fileExists(filePath: String): Boolean + + public suspend fun makeDir(filePath: String): Unit + + public suspend fun copyFile( + sourcePath: String, + targetPath: String, + ): Unit + + public fun getUserStorageDirectory(): String +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt new file mode 100644 index 00000000..d5f91955 --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt @@ -0,0 +1,25 @@ +package com.powersync.attachments + +/** + * Adapter for interfacing with remote attachment storage. + */ +public interface RemoteStorageAdapter { + /** + * Upload a file to remote storage + */ + public suspend fun uploadFile( + filename: String, + file: ByteArray, + mediaType: String, + ): Unit + + /** + * Download a file from remote storage + */ + public suspend fun downloadFile(filename: String): ByteArray + + /** + * Delete a file from remote storage + */ + public suspend fun deleteFile(filename: String) +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/SyncErrorHandler.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/SyncErrorHandler.kt new file mode 100644 index 00000000..3d8b262c --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/SyncErrorHandler.kt @@ -0,0 +1,32 @@ +package com.powersync.attachments + +/** + * Handles attachment operation errors. + * The handlers here specify if the corresponding operations should be retried. + * Attachment records are archived if an operation failed and should not be retried. + */ +public interface SyncErrorHandler { + /** + * @returns if the provided attachment download operation should be retried + */ + public suspend fun onDownloadError( + attachment: Attachment, + exception: Exception, + ): Boolean + + /** + * @returns if the provided attachment upload operation should be retried + */ + public suspend fun onUploadError( + attachment: Attachment, + exception: Exception, + ): Boolean + + /** + * @returns if the provided attachment delete operation should be retried + */ + public suspend fun onDeleteError( + attachment: Attachment, + exception: Exception, + ): Boolean +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.kt new file mode 100644 index 00000000..fe64f8dd --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.kt @@ -0,0 +1,7 @@ +package com.powersync.attachments.storage + +import com.powersync.attachments.LocalStorageAdapter + +public expect abstract class AbstractLocalStorageAdapter() : LocalStorageAdapter { + override fun getUserStorageDirectory(): String +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt new file mode 100644 index 00000000..35e9841e --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt @@ -0,0 +1,67 @@ +package com.powersync.attachments + +import com.powersync.attachments.storage.AbstractLocalStorageAdapter +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlinx.coroutines.withContext +import kotlinx.io.Buffer +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import kotlinx.io.readByteArray + +/** + * Storage adapter for local storage using the KotlinX IO library + */ +public class IOLocalStorageAdapter : AbstractLocalStorageAdapter() { + public override suspend fun saveFile( + filePath: String, + data: ByteArray, + ): Unit = + withContext(Dispatchers.IO) { + SystemFileSystem.sink(Path(filePath)).use { + // Copy to a buffer in order to write + val buffer = Buffer() + buffer.write(data) + it.write(buffer, buffer.size) + it.flush() + } + } + + public override suspend fun readFile( + filePath: String, + mediaType: String?, + ): ByteArray = + withContext(Dispatchers.IO) { + SystemFileSystem.source(Path(filePath)).use { + it.buffered().readByteArray() + } + } + + public override suspend fun deleteFile(filePath: String): Unit = + withContext(Dispatchers.IO) { + SystemFileSystem.delete(Path(filePath)) + } + + public override suspend fun fileExists(filePath: String): Boolean = + withContext(Dispatchers.IO) { + SystemFileSystem.exists(Path(filePath)) + } + + public override suspend fun makeDir(filePath: String): Unit = + withContext(Dispatchers.IO) { + SystemFileSystem.createDirectories(Path(filePath)) + } + + public override suspend fun copyFile( + sourcePath: String, + targetPath: String, + ): Unit = + withContext(Dispatchers.IO) { + SystemFileSystem.source(Path(sourcePath)).use { source -> + SystemFileSystem.sink(Path(targetPath)).use { sink -> + source.buffered().transferTo(sink.buffered()) + } + } + } +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt new file mode 100644 index 00000000..220ffece --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt @@ -0,0 +1,231 @@ +package com.powersync.attachments.sync + +import co.touchlab.kermit.Logger +import com.powersync.PowerSyncException +import com.powersync.attachments.Attachment +import com.powersync.attachments.AttachmentService +import com.powersync.attachments.AttachmentState +import com.powersync.attachments.LocalStorageAdapter +import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.SyncErrorHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlin.time.Duration + +/** + * Service used to sync attachments between local and remote storage + */ +internal class SyncingService( + private val remoteStorage: RemoteStorageAdapter, + private val localStorage: LocalStorageAdapter, + private val attachmentsService: AttachmentService, + private val getLocalUri: suspend (String) -> String, + private val errorHandler: SyncErrorHandler?, + private val logger: Logger, +) { + private val scope = CoroutineScope(Dispatchers.IO) + private val mutex = Mutex() + private val syncJob: Job + private var periodicSyncTrigger: Job? = null + + /** + * Used to trigger the sync process either manually or periodically + */ + private val syncTriggerFlow = MutableSharedFlow(replay = 0) + + init { + syncJob = + scope.launch { + merge( + // Handles manual triggers for sync events + syncTriggerFlow.asSharedFlow(), + // Triggers the sync process whenever an underlaying change to the + // attachments table happens + attachmentsService + .watchActiveAttachments(), + ) + // We only use these flows to trigger the process. We can skip multiple invocations + // while we are processing. We will always process on the trailing edge. + // This buffer operation should automatically be applied to all merged sources. + .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) + .collect { + /** + * Gets and performs the operations for active attachments which are + * pending download, upload, or delete. + */ + try { + val attachments = attachmentsService.getActiveAttachments() + // Performs pending operations and updates attachment states + handleSync(attachments) + // Cleanup + attachmentsService.deleteArchivedAttachments() + } catch (ex: Exception) { + // Rare exceptions caught here will be swallowed and retried on the + // next tick. + logger.e("Caught exception when processing attachments $ex") + } + } + } + } + + /** + * Periodically sync attachments and delete archived attachments + */ + suspend fun startPeriodicSync(period: Duration): Unit = + mutex.withLock { + periodicSyncTrigger?.cancel() + + periodicSyncTrigger = + scope.launch { + logger.i("Periodically syncing attachments") + syncTriggerFlow.emit(Unit) + delay(period) + } + } + + /** + * Enqueues a sync operation + */ + suspend fun triggerSync() { + syncTriggerFlow.emit(Unit) + } + + suspend fun close(): Unit = + mutex.withLock { + periodicSyncTrigger?.cancel() + syncJob.cancel() + syncJob.join() + } + + /** + * Handle downloading, uploading or deleting of attachments + */ + private suspend fun handleSync(attachments: List) { + val updatedAttachments = mutableListOf() + try { + for (attachment in attachments) { + when (attachment.state) { + AttachmentState.QUEUED_DOWNLOAD.ordinal -> { + logger.i("Downloading ${attachment.filename}") + updatedAttachments.add(downloadAttachment(attachment)) + } + + AttachmentState.QUEUED_UPLOAD.ordinal -> { + logger.i("Uploading ${attachment.filename}") + updatedAttachments.add(uploadAttachment(attachment)) + } + + AttachmentState.QUEUED_DELETE.ordinal -> { + logger.i("Deleting ${attachment.filename}") + updatedAttachments.add(deleteAttachment(attachment)) + } + } + } + + // Update the state of processed attachments + attachmentsService.saveAttachments(updatedAttachments) + } catch (error: Exception) { + // We retry, on the next invocation, whenever there are errors on this level + logger.e("Error during sync: ${error.message}") + } + } + + /** + * Upload attachment from local storage to remote storage. + */ + private suspend fun uploadAttachment(attachment: Attachment): Attachment { + try { + if (attachment.localUri == null) { + throw PowerSyncException( + "No localUri for attachment $attachment", + cause = Exception("attachment.localUri == null"), + ) + } + + val attachmentPath = getLocalUri(attachment.filename) + + remoteStorage.uploadFile( + attachment.filename, + localStorage.readFile(attachmentPath), + mediaType = attachment.mediaType ?: "", + ) + logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") + return attachment.copy(state = AttachmentState.SYNCED.ordinal) + } catch (e: Exception) { + logger.e("Upload attachment error for attachment $attachment: ${e.message}") + if (errorHandler != null) { + val shouldRetry = errorHandler.onUploadError(attachment, e) + if (!shouldRetry) { + logger.i("Attachment with ID ${attachment.id} has been archived") + return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) + } + } + + // Retry the upload (same state) + return attachment + } + } + + /** + * Download attachment from remote storage and save it to local storage. + * Returns the updated state of the attachment. + */ + private suspend fun downloadAttachment(attachment: Attachment): Attachment { + val imagePath = getLocalUri(attachment.filename) + + try { + val fileBlob = remoteStorage.downloadFile(attachment.filename) + localStorage.saveFile(imagePath, fileBlob) + logger.i("Downloaded file \"${attachment.id}\"") + + // The attachment has been downloaded locally + return attachment.copy(state = AttachmentState.SYNCED.ordinal) + } catch (e: Exception) { + if (errorHandler != null) { + val shouldRetry = errorHandler.onDownloadError(attachment, e) + if (!shouldRetry) { + logger.i("Attachment with ID ${attachment.id} has been archived") + return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) + } + } + + logger.e("Download attachment error for attachment $attachment: ${e.message}") + // Return the same state, this will cause a retry + return attachment + } + } + + /** + * Delete attachment from remote, local storage and then remove it from the queue. + */ + private suspend fun deleteAttachment(attachment: Attachment): Attachment { + val fileUri = getLocalUri(attachment.filename) + try { + remoteStorage.deleteFile(attachment.filename) + localStorage.deleteFile(fileUri) + return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) + } catch (e: Exception) { + if (errorHandler != null) { + val shouldRetry = errorHandler.onDeleteError(attachment, e) + if (!shouldRetry) { + logger.i("Attachment with ID ${attachment.id} has been archived") + return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) + } + } + // We'll retry this + logger.e("Error deleting attachment: ${e.message}") + return attachment + } + } +} diff --git a/attachments/src/iosMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.ios.kt b/attachments/src/iosMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.ios.kt new file mode 100644 index 00000000..4d324d65 --- /dev/null +++ b/attachments/src/iosMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.ios.kt @@ -0,0 +1,8 @@ +package com.powersync.attachments.storage + +import com.powersync.attachments.LocalStorageAdapter +import platform.Foundation.NSHomeDirectory + +public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { + actual override fun getUserStorageDirectory(): String = NSHomeDirectory() +} diff --git a/attachments/src/iosSimulatorArm64Test/kotlin/com/powersync/attachments/testutils/TestUtils.iosSimulatorArm64.kt b/attachments/src/iosSimulatorArm64Test/kotlin/com/powersync/attachments/testutils/TestUtils.iosSimulatorArm64.kt new file mode 100644 index 00000000..67a1b81a --- /dev/null +++ b/attachments/src/iosSimulatorArm64Test/kotlin/com/powersync/attachments/testutils/TestUtils.iosSimulatorArm64.kt @@ -0,0 +1,15 @@ +package com.powersync.attachments.testutils + +import com.powersync.DatabaseDriverFactory +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem + +actual val factory: DatabaseDriverFactory + get() = DatabaseDriverFactory() + +actual fun cleanup(path: String) { + val resolved = Path(path) + if (SystemFileSystem.exists(resolved)) { + SystemFileSystem.delete(resolved) + } +} diff --git a/attachments/src/jvmMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.jvm.kt b/attachments/src/jvmMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.jvm.kt new file mode 100644 index 00000000..d7600f98 --- /dev/null +++ b/attachments/src/jvmMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.jvm.kt @@ -0,0 +1,7 @@ +package com.powersync.attachments.storage + +import com.powersync.attachments.LocalStorageAdapter + +public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { + actual override fun getUserStorageDirectory(): String = System.getProperty("user.home") +} diff --git a/attachments/src/jvmTest/kotlin/com/powersync/attachments/testutils/TestUtils.jvm.kt b/attachments/src/jvmTest/kotlin/com/powersync/attachments/testutils/TestUtils.jvm.kt new file mode 100644 index 00000000..2d558118 --- /dev/null +++ b/attachments/src/jvmTest/kotlin/com/powersync/attachments/testutils/TestUtils.jvm.kt @@ -0,0 +1,11 @@ +package com.powersync.attachments.testutils + +import com.powersync.DatabaseDriverFactory +import java.io.File + +actual val factory: DatabaseDriverFactory + get() = DatabaseDriverFactory() + +actual fun cleanup(path: String) { + File(path).delete() +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 224c0422..5f1b0146 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,6 +4,7 @@ android-minSdk = "24" android-targetSdk = "35" android-compileSdk = "35" configurationAnnotations = "0.9.5" +gradleDownloadTask = "5.5.0" java = "17" idea = "243.22562.218" # Meerkat | 2024.3.1 (see https://plugins.jetbrains.com/docs/intellij/android-studio-releases-list.html) @@ -11,6 +12,7 @@ idea = "243.22562.218" # Meerkat | 2024.3.1 (see https://plugins.jetbrains.com/d kermit = "2.0.5" kotlin = "2.1.10" coroutines = "1.8.1" +kotlinGradlePlugin = "1.9.22" kotlinx-datetime = "0.6.2" kotlinx-io = "0.5.4" ktor = "3.0.1" @@ -54,8 +56,10 @@ junitVersion = "1.2.1" [libraries] configuration-annotations = { module = "co.touchlab.skie:configuration-annotations", version.ref = "configurationAnnotations" } +gradle-download-task = { module = "de.undercouch:gradle-download-task", version.ref = "gradleDownloadTask" } kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } kermit-test = { module = "co.touchlab:kermit-test", version.ref = "kermit" } +kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlinGradlePlugin" } powersync-sqlite-core-android = { module = "co.powersync:powersync-sqlite-core", version.ref = "powersync-core" } mavenPublishPlugin = { module = "com.vanniktech:gradle-maven-publish-plugin", version.ref = "maven-publish" } diff --git a/plugins/build-plugin/build.gradle.kts b/plugins/build-plugin/build.gradle.kts new file mode 100644 index 00000000..1690f112 --- /dev/null +++ b/plugins/build-plugin/build.gradle.kts @@ -0,0 +1,16 @@ +plugins { + `kotlin-dsl` // Enables Kotlin DSL for writing Gradle build logic +} + +gradlePlugin { + // Define the plugin + val sonatypeCentralUpload by plugins.creating { + id = "com.powersync.plugins.sharedbuild" + implementationClass = "com.powersync.plugins.sharedbuild.SharedBuildPlugin" + } +} + +dependencies { + implementation(libs.gradle.download.task) + implementation(libs.kotlin.gradle.plugin) +} diff --git a/plugins/build-plugin/settings.gradle.kts b/plugins/build-plugin/settings.gradle.kts new file mode 100644 index 00000000..7fbbd448 --- /dev/null +++ b/plugins/build-plugin/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "build-logic" diff --git a/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt b/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt new file mode 100644 index 00000000..302740c7 --- /dev/null +++ b/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt @@ -0,0 +1,74 @@ +package com.powersync.plugins.sharedbuild + +import de.undercouch.gradle.tasks.download.Download +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.artifacts.VersionCatalogsExtension +import org.gradle.api.tasks.Copy +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget +import org.jetbrains.kotlin.konan.target.Family + +class SharedBuildPlugin : Plugin { + override fun apply(project: Project) { + val binariesFolder = project.layout.buildDirectory.dir("binaries") + + val coreVersion = + project.extensions + .getByType(VersionCatalogsExtension::class.java) + .named("libs") + .findVersion("powersync.core") + .get() + .toString() + + val frameworkUrl = + "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/powersync-sqlite-core.xcframework.zip" + + val downloadPowersyncFramework = + project.tasks.register("downloadPowersyncFramework", Download::class.java) { + src(frameworkUrl) + dest(binariesFolder.map { it.file("framework/powersync-sqlite-core.xcframework.zip") }) + onlyIfModified(true) + } + + project.tasks.register("unzipPowersyncFramework", Copy::class.java) { + dependsOn(downloadPowersyncFramework) + + from( + project.zipTree(downloadPowersyncFramework.get().dest).matching { + include("powersync-sqlite-core.xcframework/**") + }, + ) + into(binariesFolder.map { it.dir("framework") }) + } + + project.extensions + .getByType(KotlinMultiplatformExtension::class.java) + .targets + .withType() + .configureEach { + if (konanTarget.family == Family.IOS && + konanTarget.name.contains( + "simulator", + ) + ) { + binaries + .withType() + .configureEach { + linkTaskProvider.configure { dependsOn("unzipPowersyncFramework") } + linkerOpts("-framework", "powersync-sqlite-core") + + val frameworkRoot = + binariesFolder + .map { it.dir("framework/powersync-sqlite-core.xcframework/ios-arm64_x86_64-simulator") } + .get() + .asFile.path + + linkerOpts("-F", frameworkRoot) + linkerOpts("-rpath", frameworkRoot) + } + } + } + } +} diff --git a/plugins/settings.gradle.kts b/plugins/settings.gradle.kts index 5b507f9a..cfc24791 100644 --- a/plugins/settings.gradle.kts +++ b/plugins/settings.gradle.kts @@ -22,4 +22,5 @@ dependencyResolutionManagement { rootProject.name = "plugins" -include(":sonatype") \ No newline at end of file +include(":sonatype") +include(":build-plugin") diff --git a/settings.gradle.kts b/settings.gradle.kts index b7472215..1e529c5a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -27,5 +27,6 @@ include(":persistence") include(":PowerSyncKotlin") include(":compose") +include(":attachments") enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") From f3ff250029963b521877c38fb4cc5cc5a33f9ba5 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 1 Apr 2025 17:46:42 +0200 Subject: [PATCH 02/51] wip --- .../powersync/attachments/AttachmentsTest.kt | 62 ++++++++++++++++++- .../testutils/MockedRemoteStorage.kt | 17 +++++ .../testutils/TestAttachmentsQueue.kt | 29 +++++++++ .../AbstractAttachmentQueue.kt | 12 ++-- .../com.powersync.attachments/Attachment.kt | 8 ++- .../AttachmentService.kt | 34 ++++++---- .../AttachmentTable.kt | 26 ++++++++ .../sync/SyncingService.kt | 6 +- core/build.gradle.kts | 44 +------------ .../db/internal/InternalDatabaseImpl.kt | 1 + 10 files changed, 176 insertions(+), 63 deletions(-) create mode 100644 attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt create mode 100644 attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt create mode 100644 attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentTable.kt diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt index 899ac86b..86d5dae7 100644 --- a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt +++ b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt @@ -1,14 +1,20 @@ package com.powersync.attachments +import app.cash.turbine.turbineScope import co.touchlab.kermit.ExperimentalKermitApi import com.powersync.PowerSyncDatabase +import com.powersync.attachments.testutils.MockedRemoteStorage +import com.powersync.attachments.testutils.TestAttachmentsQueue import com.powersync.attachments.testutils.UserRow import com.powersync.db.schema.Schema +import dev.mokkery.spy +import dev.mokkery.verifySuspend import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import kotlin.test.AfterTest import kotlin.test.BeforeTest import kotlin.test.Test +import kotlin.test.assertEquals @OptIn(ExperimentalKermitApi::class) class AttachmentsTest { @@ -17,7 +23,7 @@ class AttachmentsTest { private fun openDB() = PowerSyncDatabase( factory = com.powersync.attachments.testutils.factory, - schema = Schema(UserRow.table), + schema = Schema(UserRow.table, createAttachmentsTable("attachments")), dbFilename = "testdb", ) @@ -47,4 +53,58 @@ class AttachmentsTest { runTest { database.get("SELECT powersync_rs_version();") { it.getString(0)!! } } + + @Test + fun testAttachmentDownload() = + runTest { + turbineScope { + val remote = spy(MockedRemoteStorage()) + + val queue = + TestAttachmentsQueue(db = database, remoteStorage = remote) + + queue.start() + + // Monitor the attachments table for testing + val attachmentQuery = + database + .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } + .testIn(this) + + val result = attachmentQuery.awaitItem() + + // There should not be any attachment records here + assertEquals(expected = 0, actual = result.size) + + // Create a user with a photo_id specified. + // This code did not save an attachment before assigning a photo_id. + // This is equivalent to requiring an attachment download + database.execute( + """ + INSERT INTO + users (id, name, email, photo_id) + VALUES + (uuid(), "steven", "steven@journeyapps.com", uuid()) + """, + ) + +// The watched query should cause the attachment record to be pending download + val afterInsert = attachmentQuery.awaitItem() + + assertEquals( + expected = 1, + actual = afterInsert.size, + "Should contain 1 attachment record", + ) + + val item = afterInsert.first() + assertEquals(expected = AttachmentState.QUEUED_DOWNLOAD.ordinal, item.state) + + // A download should have been attempted for this file + verifySuspend { remote.downloadFile(item.filename) } + + attachmentQuery.cancel() + queue.close() + } + } } diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt new file mode 100644 index 00000000..e43129c8 --- /dev/null +++ b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt @@ -0,0 +1,17 @@ +package com.powersync.attachments.testutils + +import com.powersync.attachments.RemoteStorageAdapter + +class MockedRemoteStorage : RemoteStorageAdapter { + override suspend fun uploadFile( + filename: String, + file: ByteArray, + mediaType: String, + ) { + } + + override suspend fun downloadFile(filename: String): ByteArray = ByteArray(1) + + override suspend fun deleteFile(filename: String) { + } +} diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt new file mode 100644 index 00000000..f5c5ed6d --- /dev/null +++ b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt @@ -0,0 +1,29 @@ +package com.powersync.attachments.testutils + +import com.powersync.PowerSyncDatabase +import com.powersync.attachments.AbstractAttachmentQueue +import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.WatchedAttachmentItem +import com.powersync.db.getString +import kotlinx.coroutines.flow.Flow + +internal class TestAttachmentsQueue( + db: PowerSyncDatabase, + remoteStorage: RemoteStorageAdapter, +) : AbstractAttachmentQueue(db, remoteStorage) { + override fun watchAttachments(): Flow> = + db.watch( + sql = + """ + SELECT + id, + photo_id + FROM + users + WHERE + photo_id IS NOT NULL + """, + ) { + WatchedAttachmentItem(id = it.getString("id"), fileExtension = "jpg") + } +} diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt index a4ccdef6..d3ecc515 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt @@ -53,15 +53,15 @@ public abstract class AbstractAttachmentQueue( /** * PowerSync database client */ - private val db: PowerSyncDatabase, + public val db: PowerSyncDatabase, /** * Adapter which interfaces with the remote storage backend */ - private val remoteStorage: RemoteStorageAdapter, + public val remoteStorage: RemoteStorageAdapter, /** * Provides access to local filesystem storage methods */ - private val localStorage: LocalStorageAdapter = IOLocalStorageAdapter(), + public val localStorage: LocalStorageAdapter = IOLocalStorageAdapter(), /** * Directory where attachment files will be written to disk */ @@ -74,7 +74,7 @@ public abstract class AbstractAttachmentQueue( * Attachment operation error handler. This specified if failed attachment operations * should be retried. */ - private val errorHandler: SyncErrorHandler?, + private val errorHandler: SyncErrorHandler? = null, /** * Periodic interval to trigger attachment sync operations */ @@ -86,7 +86,7 @@ public abstract class AbstractAttachmentQueue( /** * Logging interface used for all log operations */ - private val logger: Logger = Logger, + public val logger: Logger = Logger, ) { public companion object { public const val DEFAULT_TABLE_NAME: String = "attachments" @@ -271,6 +271,8 @@ public abstract class AbstractAttachmentQueue( .forEach { attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED.ordinal)) } + + attachmentsService.saveAttachments(attachmentUpdates) } /** diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt index 548e8d9e..f9cf0017 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt @@ -2,7 +2,9 @@ package com.powersync.attachments import com.powersync.db.SqlCursor import com.powersync.db.getLong +import com.powersync.db.getLongOptional import com.powersync.db.getString +import com.powersync.db.getStringOptional /** * Enum for the attachment state @@ -33,9 +35,9 @@ public data class Attachment( id = cursor.getString(name = "id"), timestamp = cursor.getLong("timestamp"), filename = cursor.getString(name = "filename"), - localUri = cursor.getString(name = "local_uri"), - mediaType = cursor.getString(name = "media_type"), - size = cursor.getLong("size"), + localUri = cursor.getStringOptional(name = "local_uri"), + mediaType = cursor.getStringOptional(name = "media_type"), + size = cursor.getLongOptional("size"), state = cursor.getLong("state").toInt(), ) } diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt index 9f82f4d8..57d09c55 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt @@ -33,8 +33,8 @@ public class AttachmentService( */ public suspend fun ignoreAttachment(id: String) { db.execute( - "UPDATE $table SET state = ${AttachmentState.ARCHIVED.ordinal} WHERE id = ?", - listOf(id), + "UPDATE $table SET state = ? WHERE id = ?", + listOf(AttachmentState.ARCHIVED.ordinal, id), ) } @@ -88,16 +88,20 @@ public class AttachmentService( public suspend fun getActiveAttachments(): List = db.getAll( """ - SELECT + SELECT * FROM $table WHERE - state = ${AttachmentState.QUEUED_DOWNLOAD.ordinal} - OR state = ${AttachmentState.QUEUED_DELETE.ordinal} - OR state = ${AttachmentState.QUEUED_UPLOAD.ordinal} + state = ? + OR state = ? + OR state = ? """, - listOf(AttachmentState.ARCHIVED.ordinal), + listOf( + AttachmentState.QUEUED_UPLOAD.ordinal, + AttachmentState.QUEUED_DOWNLOAD.ordinal, + AttachmentState.QUEUED_DELETE.ordinal, + ), ) { Attachment.fromCursor(it) } /** @@ -114,11 +118,16 @@ public class AttachmentService( FROM $table WHERE - state = ${AttachmentState.QUEUED_DOWNLOAD.ordinal} - OR state = ${AttachmentState.QUEUED_DELETE.ordinal} - OR state = ${AttachmentState.QUEUED_UPLOAD.ordinal} + state = ? + OR state = ? + OR state = ? """, - ) { } + listOf( + AttachmentState.QUEUED_UPLOAD.ordinal, + AttachmentState.QUEUED_DOWNLOAD.ordinal, + AttachmentState.QUEUED_DELETE.ordinal, + ), + ) { it.getString(0)!! } // We only use changes here to trigger a sync consolidation .map { Unit } } @@ -139,8 +148,9 @@ public class AttachmentService( db.execute( """ DELETE FROM $table - WHERE state = ${AttachmentState.ARCHIVED.ordinal} + WHERE state = ? """, + listOf(AttachmentState.ARCHIVED.ordinal), ) } diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentTable.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentTable.kt new file mode 100644 index 00000000..78adbbbb --- /dev/null +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentTable.kt @@ -0,0 +1,26 @@ +package com.powersync.attachments + +import com.powersync.db.schema.Column +import com.powersync.db.schema.ColumnType +import com.powersync.db.schema.Table + +/** + * Creates a PowerSync table for storing local attachment state + */ +public fun createAttachmentsTable( + name: String, + additionalColumns: List? = null, +): Table = + Table( + name = name, + columns = + listOf( + Column("filename", ColumnType.TEXT), + Column("local_uri", ColumnType.TEXT), + Column("timestamp", ColumnType.INTEGER), + Column("size", ColumnType.INTEGER), + Column("media_type", ColumnType.TEXT), + Column("state", ColumnType.INTEGER), + ).plus(additionalColumns ?: emptyList()), + localOnly = true, + ) diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt index 220ffece..68461f26 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt @@ -8,6 +8,7 @@ import com.powersync.attachments.AttachmentState import com.powersync.attachments.LocalStorageAdapter import com.powersync.attachments.RemoteStorageAdapter import com.powersync.attachments.SyncErrorHandler +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO @@ -58,7 +59,7 @@ internal class SyncingService( // We only use these flows to trigger the process. We can skip multiple invocations // while we are processing. We will always process on the trailing edge. // This buffer operation should automatically be applied to all merged sources. - .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) + .buffer(3, onBufferOverflow = BufferOverflow.DROP_OLDEST) .collect { /** * Gets and performs the operations for active attachments which are @@ -71,6 +72,9 @@ internal class SyncingService( // Cleanup attachmentsService.deleteArchivedAttachments() } catch (ex: Exception) { + if (ex is CancellationException) { + throw ex + } // Rare exceptions caught here will be swallowed and retried on the // next tick. logger.e("Caught exception when processing attachments $ex") diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 96f77274..b68d9937 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -5,10 +5,8 @@ import org.gradle.internal.os.OperatingSystem import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget -import org.jetbrains.kotlin.gradle.plugin.mpp.TestExecutable import org.jetbrains.kotlin.gradle.targets.jvm.tasks.KotlinJvmTest import org.jetbrains.kotlin.gradle.tasks.KotlinTest -import org.jetbrains.kotlin.konan.target.Family plugins { @@ -19,6 +17,7 @@ plugins { alias(libs.plugins.downloadPlugin) alias(libs.plugins.kotlinter) id("com.powersync.plugins.sonatype") + id("com.powersync.plugins.sharedbuild") alias(libs.plugins.mokkery) alias(libs.plugins.kotlin.atomicfu) } @@ -72,29 +71,6 @@ val downloadPowersyncDesktopBinaries by tasks.registering(Download::class) { onlyIfModified(true) } -val downloadPowersyncFramework by tasks.registering(Download::class) { - val coreVersion = - libs.versions.powersync.core - .get() - val framework = - "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/powersync-sqlite-core.xcframework.zip" - - src(framework) - dest(binariesFolder.map { it.file("framework/powersync-sqlite-core.xcframework.zip") }) - onlyIfModified(true) -} - -val unzipPowersyncFramework by tasks.registering(Copy::class) { - dependsOn(downloadPowersyncFramework) - - from( - zipTree(downloadPowersyncFramework.get().dest).matching { - include("powersync-sqlite-core.xcframework/**") - }, - ) - into(binariesFolder.map { it.dir("framework") }) -} - val sqliteJDBCFolder = project.layout.buildDirectory .dir("jdbc") @@ -178,20 +154,6 @@ kotlin { } } - if (konanTarget.family == Family.IOS && konanTarget.name.contains("simulator")) { - binaries.withType().configureEach { - linkTaskProvider.configure { dependsOn(unzipPowersyncFramework) } - linkerOpts("-framework", "powersync-sqlite-core") - val frameworkRoot = - binariesFolder - .map { it.dir("framework/powersync-sqlite-core.xcframework/ios-arm64_x86_64-simulator") } - .get() - .asFile.path - - linkerOpts("-F", frameworkRoot) - linkerOpts("-rpath", frameworkRoot) - } - } /* If we ever need macOS support: { @@ -315,8 +277,8 @@ android { } androidComponents.onVariants { - tasks.named("preBuild") { - dependsOn(moveJDBCJNIFiles) + tasks.named("preBuild") { + dependsOn(moveJDBCJNIFiles) } } diff --git a/core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt b/core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt index ca577ca4..ff0de31c 100644 --- a/core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt @@ -281,6 +281,7 @@ internal fun getBindersFromParams(parameters: List?): (SqlPreparedStatemen is Boolean -> bindBoolean(index, parameter) is String -> bindString(index, parameter) is Long -> bindLong(index, parameter) + is Int -> bindLong(index, parameter.toLong()) is Double -> bindDouble(index, parameter) is ByteArray -> bindBytes(index, parameter) else -> { From dbcea9f3db5458dab05cae3633ea0966939a6dcf Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 2 Apr 2025 13:04:21 +0200 Subject: [PATCH 03/51] wip tests --- attachments/build.gradle.kts | 2 + .../powersync/attachments/AttachmentsTest.kt | 168 ++++++++++++++++-- .../testutils/MockedRemoteStorage.kt | 13 +- .../testutils/TestAttachmentsQueue.kt | 6 +- .../AbstractAttachmentQueue.kt | 73 ++++---- .../AttachmentService.kt | 40 ++++- .../LocalStorageAdapter.kt | 12 +- .../RemoteStorageAdapter.kt | 8 +- .../storage/IOLocalStorageAdapter.kt | 47 +++-- .../sync/SyncingService.kt | 41 +++-- 10 files changed, 326 insertions(+), 84 deletions(-) diff --git a/attachments/build.gradle.kts b/attachments/build.gradle.kts index a18df883..d03f2c3d 100644 --- a/attachments/build.gradle.kts +++ b/attachments/build.gradle.kts @@ -100,6 +100,8 @@ kotlin { implementation(libs.kermit.test) implementation(libs.ktor.client.mock) implementation(libs.test.turbine) + // Allows using the core test utils + implementation(project(":core")) } jvmTest.get().dependsOn(commonIntegrationTest) diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt index 86d5dae7..bd23acfb 100644 --- a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt +++ b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt @@ -7,14 +7,19 @@ import com.powersync.attachments.testutils.MockedRemoteStorage import com.powersync.attachments.testutils.TestAttachmentsQueue import com.powersync.attachments.testutils.UserRow import com.powersync.db.schema.Schema +import dev.mokkery.matcher.any import dev.mokkery.spy import dev.mokkery.verifySuspend +import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import kotlin.test.AfterTest import kotlin.test.BeforeTest import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue @OptIn(ExperimentalKermitApi::class) class AttachmentsTest { @@ -61,7 +66,12 @@ class AttachmentsTest { val remote = spy(MockedRemoteStorage()) val queue = - TestAttachmentsQueue(db = database, remoteStorage = remote) + TestAttachmentsQueue( + db = database, + remoteStorage = remote, + // For these tests (jvm/ios) this should be fine for now + attachmentDirectoryName = "/tmp", + ) queue.start() @@ -88,20 +98,158 @@ class AttachmentsTest { """, ) -// The watched query should cause the attachment record to be pending download - val afterInsert = attachmentQuery.awaitItem() + var attachmentRecord = attachmentQuery.awaitItem().first() + assertNotNull( + attachmentRecord, + """ + An attachment record should be created after creating a user with a photo_id + " + """.trimIndent(), + ) - assertEquals( - expected = 1, - actual = afterInsert.size, - "Should contain 1 attachment record", + /** + * The timing of the watched query resolving might differ slightly. + * We might get a watched query result where the attachment is QUEUED_DOWNLOAD + * or we could get the result once it has been DOWNLOADED. + * We should assert that the download happens eventually. + */ + + if (attachmentRecord.state == AttachmentState.QUEUED_DOWNLOAD.ordinal) { + // Wait for the download to be triggered + attachmentRecord = attachmentQuery.awaitItem().first() + } + + assertEquals(expected = AttachmentState.SYNCED.ordinal, attachmentRecord.state) + + // A download should have been attempted for this file + verifySuspend { remote.downloadFile(attachmentRecord.filename) } + + // A file should now exist + val localUri = attachmentRecord.localUri!! + assertTrue { queue.localStorage.fileExists(localUri) } + + // Now clear the user's photo_id. The attachment should be archived + database.execute( + """ + UPDATE + users + SET + photo_id = NULL + """, ) - val item = afterInsert.first() - assertEquals(expected = AttachmentState.QUEUED_DOWNLOAD.ordinal, item.state) + /** + * The timing of the watched query resolving might differ slightly. + * We might get a watched query result where the attachment is ARCHIVED + * or we could get the result once it has been deleted. + * The file should be deleted eventually + */ + var nextRecord: Attachment? = attachmentQuery.awaitItem().first() + if (nextRecord?.state == AttachmentState.ARCHIVED.ordinal) { + nextRecord = attachmentQuery.awaitItem().getOrNull(0) + } + + // The record should have been deleted + assertNull(nextRecord) + + // The file should have been deleted from storage + assertEquals(expected = false, actual = queue.localStorage.fileExists(localUri)) + + attachmentQuery.cancel() + queue.close() + } + } + + @Test + fun testAttachmentUpload() = + runTest { + turbineScope { + val remote = spy(MockedRemoteStorage()) + + val queue = + TestAttachmentsQueue( + db = database, + remoteStorage = remote, + // For these tests (jvm/ios) this should be fine for now + attachmentDirectoryName = "/tmp", + ) + + queue.start() + + // Monitor the attachments table for testing + val attachmentQuery = + database + .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } + .testIn(this) + + val result = attachmentQuery.awaitItem() + + // There should not be any attachment records here + assertEquals(expected = 0, actual = result.size) + + /** + * Creates an attachment given a flow of bytes (the file data) then assigns this to + * a newly created user. + */ + queue.saveFile(flowOf(ByteArray(1)), "image/jpg", "jpg") { tx, attachment -> + // Set the photo_id of a new user to the attachment id + tx.execute( + """ + INSERT INTO + users (id, name, email, photo_id) + VALUES + (uuid(), "steven", "steven@steven.com", ?) + """, + listOf(attachment.id), + ) + } + + var attachmentRecord = attachmentQuery.awaitItem().first() + assertNotNull(attachmentRecord) + + if (attachmentRecord.state == AttachmentState.QUEUED_UPLOAD.ordinal) { + // Wait for it to be synced + attachmentRecord = attachmentQuery.awaitItem().first() + } + + assertEquals( + expected = AttachmentState.SYNCED.ordinal, + attachmentRecord.state, + ) // A download should have been attempted for this file - verifySuspend { remote.downloadFile(item.filename) } + verifySuspend { + remote.uploadFile( + attachmentRecord.filename, + any(), + attachmentRecord.mediaType, + ) + } + + // A file should now exist + val localUri = attachmentRecord.localUri!! + assertTrue { queue.localStorage.fileExists(localUri) } + + // Now clear the user's photo_id. The attachment should be archived + database.execute( + """ + UPDATE + users + SET + photo_id = NULL + """, + ) + + var nextRecord: Attachment? = attachmentQuery.awaitItem().first() + if (nextRecord?.state == AttachmentState.ARCHIVED.ordinal) { + nextRecord = attachmentQuery.awaitItem().getOrNull(0) + } + + // The record should have been deleted + assertNull(nextRecord) + + // The file should have been deleted from storage + assertEquals(expected = false, actual = queue.localStorage.fileExists(localUri)) attachmentQuery.cancel() queue.close() diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt index e43129c8..20a39940 100644 --- a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt +++ b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt @@ -1,17 +1,24 @@ package com.powersync.attachments.testutils import com.powersync.attachments.RemoteStorageAdapter +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow class MockedRemoteStorage : RemoteStorageAdapter { override suspend fun uploadFile( filename: String, - file: ByteArray, - mediaType: String, + file: Flow, + mediaType: String?, ) { + // No op } - override suspend fun downloadFile(filename: String): ByteArray = ByteArray(1) + override suspend fun downloadFile(filename: String): Flow = + flow { + emit(ByteArray(1)) + } override suspend fun deleteFile(filename: String) { + // No op } } diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt index f5c5ed6d..c1fee706 100644 --- a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt +++ b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt @@ -10,13 +10,13 @@ import kotlinx.coroutines.flow.Flow internal class TestAttachmentsQueue( db: PowerSyncDatabase, remoteStorage: RemoteStorageAdapter, -) : AbstractAttachmentQueue(db, remoteStorage) { + attachmentDirectoryName: String, +) : AbstractAttachmentQueue(db, remoteStorage, attachmentDirectoryName = attachmentDirectoryName) { override fun watchAttachments(): Flow> = db.watch( sql = """ SELECT - id, photo_id FROM users @@ -24,6 +24,6 @@ internal class TestAttachmentsQueue( photo_id IS NOT NULL """, ) { - WatchedAttachmentItem(id = it.getString("id"), fileExtension = "jpg") + WatchedAttachmentItem(id = it.getString("photo_id"), fileExtension = "jpg") } } diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt index d3ecc515..98aeea85 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt @@ -4,6 +4,7 @@ import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase import com.powersync.PowerSyncException import com.powersync.attachments.sync.SyncingService +import com.powersync.db.internal.ConnectionContext import com.powersync.db.runWrappedSuspending import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -278,42 +279,52 @@ public abstract class AbstractAttachmentQueue( /** * A function which creates a new attachment locally. This new attachment is queued for upload * after creation. - * The relevant attachment file should be persisted to disk before calling this method. - * The default implementation assumes the attachment file has been written to the path - * ```kotlin - * val path = getLocalFilePathSuffix( - * resolveNewAttachmentFilename( - * attachmentId = attachmentId, - * fileExtension = fileExtension, - * )) - * ) - * ``` - * This method can be overriden for custom behaviour. + * + * The filename is resolved using [resolveNewAttachmentFilename]. + * + * A [updateHook] is provided which should be used when assigning relationships to the newly + * created attachment. This hook is executed in the same writeTransaction which creates the + * attachment record. + * + * This method can be overriden for custom behaviour. */ @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun saveFile( - attachmentId: String, - size: Long, + public suspend fun saveFile( + data: Flow, mediaType: String, fileExtension: String?, - ): Attachment = + updateHook: (context: ConnectionContext, attachment: Attachment) -> R, + ): R = runWrappedSuspending { + val id = db.get("SELECT uuid()") { it.getString(0)!! } val filename = - resolveNewAttachmentFilename( - attachmentId = attachmentId, - fileExtension = fileExtension, - ) + resolveNewAttachmentFilename(attachmentId = id, fileExtension = fileExtension) + val localUri = getLocalUri(filename) - return@runWrappedSuspending attachmentsService.saveAttachment( - Attachment( - id = attachmentId, - filename = filename, - size = size, - mediaType = mediaType, - state = AttachmentState.QUEUED_UPLOAD.ordinal, - localUri = getLocalFilePathSuffix(filename).toString(), - ), - ) + // write the file to the filesystem + val fileSize = localStorage.saveFile(localUri, data) + + /** + * Starts a write transaction. The attachment record and relevant local relationship + * assignment should happen in the same transaction. + */ + db.writeTransaction { tx -> + val attachment = + attachmentsService.upsertAttachment( + Attachment( + id = id, + filename = filename, + size = fileSize, + mediaType = mediaType, + state = AttachmentState.QUEUED_UPLOAD.ordinal, + localUri = localUri, + ), + tx, + ) + + // Allow consumers to set relationships to this attachment id + updateHook(tx, attachment) + } } /** @@ -354,8 +365,8 @@ public abstract class AbstractAttachmentQueue( * Return users storage directory with the attachmentPath use to load the file. * Example: filePath: "attachments/attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg" */ - public fun getLocalUri(filePath: String): String { + public fun getLocalUri(filename: String): String { val storageDirectory = getStorageDirectory() - return Path(storageDirectory, filePath).toString() + return Path(storageDirectory, filename).toString() } } diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt index 57d09c55..5a2868a2 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt @@ -6,6 +6,8 @@ import com.powersync.db.internal.ConnectionContext import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map import kotlinx.datetime.Clock +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json /** * Service for interacting with the local attachment records. @@ -79,7 +81,16 @@ public class AttachmentService( public suspend fun getAttachments(): List = db.getAll( - "SELECT * FROM $table WHERE id IS NOT NULL", + """ + SELECT + * + FROM + $table + WHERE + id IS NOT NULL + ORDER BY + timestamp ASC + """, ) { Attachment.fromCursor(it) } /** @@ -96,6 +107,8 @@ public class AttachmentService( state = ? OR state = ? OR state = ? + ORDER BY + timestamp ASC """, listOf( AttachmentState.QUEUED_UPLOAD.ordinal, @@ -121,6 +134,8 @@ public class AttachmentService( state = ? OR state = ? OR state = ? + ORDER BY + timestamp ASC """, listOf( AttachmentState.QUEUED_UPLOAD.ordinal, @@ -144,17 +159,26 @@ public class AttachmentService( /** * Delete attachments which have been archived */ - public suspend fun deleteArchivedAttachments() { + public suspend fun deleteArchivedAttachments(callback: suspend (attachments: List) -> Unit) { + // First fetch the attachments in order to allow other cleanup + val attachments = + db.getAll( + "SELECT * FROM $table WHERE state = ?", + listOf(AttachmentState.ARCHIVED.ordinal), + ) { Attachment.fromCursor(it) } + callback(attachments) db.execute( - """ - DELETE FROM $table - WHERE state = ? - """, - listOf(AttachmentState.ARCHIVED.ordinal), + "DELETE FROM $table WHERE id IN (SELECT value FROM json_each(?));", + listOf( + Json.encodeToString(attachments.map { it.id }), + ), ) } - private fun upsertAttachment( + /** + * Upserts an attachment record synchronously given a database connection context. + */ + public fun upsertAttachment( attachment: Attachment, context: ConnectionContext, ): Attachment { diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt index ad830c8e..dcc66f79 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt @@ -1,18 +1,24 @@ package com.powersync.attachments +import kotlinx.coroutines.flow.Flow + /** * Storage adapter for local storage */ public interface LocalStorageAdapter { + /** + * Saves a source of data bytes to a path. + * @returns the bytesize of the file + */ public suspend fun saveFile( filePath: String, - data: ByteArray, - ): Unit + data: Flow, + ): Long public suspend fun readFile( filePath: String, mediaType: String? = null, - ): ByteArray + ): Flow public suspend fun deleteFile(filePath: String): Unit diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt index d5f91955..8d21e1e3 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt @@ -1,5 +1,7 @@ package com.powersync.attachments +import kotlinx.coroutines.flow.Flow + /** * Adapter for interfacing with remote attachment storage. */ @@ -9,14 +11,14 @@ public interface RemoteStorageAdapter { */ public suspend fun uploadFile( filename: String, - file: ByteArray, - mediaType: String, + file: Flow, + mediaType: String?, ): Unit /** * Download a file from remote storage */ - public suspend fun downloadFile(filename: String): ByteArray + public suspend fun downloadFile(filename: String): Flow /** * Delete a file from remote storage diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt index 35e9841e..382ed390 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt @@ -1,14 +1,18 @@ package com.powersync.attachments import com.powersync.attachments.storage.AbstractLocalStorageAdapter +import io.ktor.utils.io.core.remaining import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow import kotlinx.coroutines.withContext import kotlinx.io.Buffer import kotlinx.io.buffered import kotlinx.io.files.Path import kotlinx.io.files.SystemFileSystem import kotlinx.io.readByteArray +import kotlin.math.min /** * Storage adapter for local storage using the KotlinX IO library @@ -16,25 +20,46 @@ import kotlinx.io.readByteArray public class IOLocalStorageAdapter : AbstractLocalStorageAdapter() { public override suspend fun saveFile( filePath: String, - data: ByteArray, - ): Unit = + data: Flow, + ): Long = withContext(Dispatchers.IO) { - SystemFileSystem.sink(Path(filePath)).use { + var totalSize = 0L + SystemFileSystem.sink(Path(filePath)).use { sink -> // Copy to a buffer in order to write - val buffer = Buffer() - buffer.write(data) - it.write(buffer, buffer.size) - it.flush() + Buffer().use { buffer -> + data.collect { chunk -> + // Copy into a buffer in order to sink the chunk + buffer.write(chunk, 0) + val chunkSize = chunk.size.toLong() + totalSize += chunkSize + sink.write(buffer, chunkSize) + } + } + sink.flush() + return@withContext totalSize } } public override suspend fun readFile( filePath: String, mediaType: String?, - ): ByteArray = - withContext(Dispatchers.IO) { - SystemFileSystem.source(Path(filePath)).use { - it.buffered().readByteArray() + ): Flow = + flow { + withContext(Dispatchers.IO) { + SystemFileSystem.source(Path(filePath)).use { source -> + source.buffered().use { bufferedSource -> + val bufferSize = 8192L // Read in 8KB chunks + while (bufferedSource.remaining > 0) { + val byteCount = + min( + bufferedSource.remaining, + bufferSize, + ) + val bytesRead = bufferedSource.readByteArray(byteCount.toInt()) + emit(bytesRead) + } + } + } } } diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt b/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt index 68461f26..050fa8ef 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt +++ b/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt @@ -59,7 +59,7 @@ internal class SyncingService( // We only use these flows to trigger the process. We can skip multiple invocations // while we are processing. We will always process on the trailing edge. // This buffer operation should automatically be applied to all merged sources. - .buffer(3, onBufferOverflow = BufferOverflow.DROP_OLDEST) + .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) .collect { /** * Gets and performs the operations for active attachments which are @@ -69,8 +69,19 @@ internal class SyncingService( val attachments = attachmentsService.getActiveAttachments() // Performs pending operations and updates attachment states handleSync(attachments) - // Cleanup - attachmentsService.deleteArchivedAttachments() + + // Cleanup archived attachments + attachmentsService.deleteArchivedAttachments { pendingDelete -> + for (attachment in pendingDelete) { + if (attachment.localUri == null) { + continue + } + if (!localStorage.fileExists(attachment.localUri)) { + continue + } + localStorage.deleteFile(attachment.localUri) + } + } } catch (ex: Exception) { if (ex is CancellationException) { throw ex @@ -157,11 +168,9 @@ internal class SyncingService( ) } - val attachmentPath = getLocalUri(attachment.filename) - remoteStorage.uploadFile( attachment.filename, - localStorage.readFile(attachmentPath), + localStorage.readFile(attachment.localUri), mediaType = attachment.mediaType ?: "", ) logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") @@ -186,15 +195,22 @@ internal class SyncingService( * Returns the updated state of the attachment. */ private suspend fun downloadAttachment(attachment: Attachment): Attachment { - val imagePath = getLocalUri(attachment.filename) + /** + * When downloading an attachment we take the filename and resolve + * the local_uri where the file will be stored + */ + val attachmentPath = getLocalUri(attachment.filename) try { - val fileBlob = remoteStorage.downloadFile(attachment.filename) - localStorage.saveFile(imagePath, fileBlob) + val fileFlow = remoteStorage.downloadFile(attachment.filename) + localStorage.saveFile(attachmentPath, fileFlow) logger.i("Downloaded file \"${attachment.id}\"") // The attachment has been downloaded locally - return attachment.copy(state = AttachmentState.SYNCED.ordinal) + return attachment.copy( + localUri = attachmentPath, + state = AttachmentState.SYNCED.ordinal, + ) } catch (e: Exception) { if (errorHandler != null) { val shouldRetry = errorHandler.onDownloadError(attachment, e) @@ -214,10 +230,11 @@ internal class SyncingService( * Delete attachment from remote, local storage and then remove it from the queue. */ private suspend fun deleteAttachment(attachment: Attachment): Attachment { - val fileUri = getLocalUri(attachment.filename) try { remoteStorage.deleteFile(attachment.filename) - localStorage.deleteFile(fileUri) + if (attachment.localUri != null) { + localStorage.deleteFile(attachment.localUri) + } return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) } catch (e: Exception) { if (errorHandler != null) { From 7de65fdd1784893fc36b7bf0e81a3225dbb98e5b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 2 Apr 2025 13:16:54 +0200 Subject: [PATCH 04/51] include as package in core module --- attachments/README.md | 0 attachments/build.gradle.kts | 153 ------------------ attachments/gradle.properties | 0 .../attachments/testutils/TestUtils.kt | 7 - .../attachments/testutils/UserRow.kt | 35 ---- .../testutils/TestUtils.iosSimulatorArm64.kt | 15 -- .../attachments/testutils/TestUtils.jvm.kt | 11 -- .../AbstractLocalStorageAdapter.android.kt | 1 + .../kotlin/com/powersync}/AttachmentsTest.kt | 17 +- .../testutils/MockedRemoteStorage.kt | 2 +- .../testutils/TestAttachmentsQueue.kt | 2 +- .../kotlin/com/powersync/testutils/UserRow.kt | 14 +- .../attachments}/AbstractAttachmentQueue.kt | 0 .../com/powersync/attachments}/Attachment.kt | 0 .../attachments}/AttachmentService.kt | 0 .../powersync/attachments}/AttachmentTable.kt | 0 .../attachments}/LocalStorageAdapter.kt | 0 .../attachments}/RemoteStorageAdapter.kt | 0 .../attachments}/SyncErrorHandler.kt | 0 .../storage/AbstractLocalStorageAdapter.kt | 1 + .../storage/IOLocalStorageAdapter.kt | 0 .../attachments}/sync/SyncingService.kt | 0 .../AbstractLocalStorageAdapter.ios.kt | 1 + .../AbstractLocalStorageAdapter.jvm.kt | 1 + 24 files changed, 29 insertions(+), 231 deletions(-) delete mode 100644 attachments/README.md delete mode 100644 attachments/build.gradle.kts delete mode 100644 attachments/gradle.properties delete mode 100644 attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestUtils.kt delete mode 100644 attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/UserRow.kt delete mode 100644 attachments/src/iosSimulatorArm64Test/kotlin/com/powersync/attachments/testutils/TestUtils.iosSimulatorArm64.kt delete mode 100644 attachments/src/jvmTest/kotlin/com/powersync/attachments/testutils/TestUtils.jvm.kt rename {attachments/src/androidMain/kotlin/com.powersync.attachments => core/src/androidMain/kotlin/com/powersync/attachments}/storage/AbstractLocalStorageAdapter.android.kt (84%) rename {attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments => core/src/commonIntegrationTest/kotlin/com/powersync}/AttachmentsTest.kt (95%) rename {attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments => core/src/commonIntegrationTest/kotlin/com/powersync}/testutils/MockedRemoteStorage.kt (92%) rename {attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments => core/src/commonIntegrationTest/kotlin/com/powersync}/testutils/TestAttachmentsQueue.kt (95%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/AbstractAttachmentQueue.kt (100%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/Attachment.kt (100%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/AttachmentService.kt (100%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/AttachmentTable.kt (100%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/LocalStorageAdapter.kt (100%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/RemoteStorageAdapter.kt (100%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/SyncErrorHandler.kt (100%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/storage/AbstractLocalStorageAdapter.kt (85%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/storage/IOLocalStorageAdapter.kt (100%) rename {attachments/src/commonMain/kotlin/com.powersync.attachments => core/src/commonMain/kotlin/com/powersync/attachments}/sync/SyncingService.kt (100%) rename {attachments/src/iosMain/kotlin/com.powersync.attachments => core/src/iosMain/kotlin/com/powersync/attachments}/storage/AbstractLocalStorageAdapter.ios.kt (83%) rename {attachments/src/jvmMain/kotlin/com.powersync.attachments => core/src/jvmMain/kotlin/com/powersync/attachments}/storage/AbstractLocalStorageAdapter.jvm.kt (82%) diff --git a/attachments/README.md b/attachments/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/attachments/build.gradle.kts b/attachments/build.gradle.kts deleted file mode 100644 index d03f2c3d..00000000 --- a/attachments/build.gradle.kts +++ /dev/null @@ -1,153 +0,0 @@ -import com.powersync.plugins.sonatype.setupGithubRepository -import org.gradle.api.tasks.testing.logging.TestExceptionFormat -import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi -import org.jetbrains.kotlin.gradle.dsl.JvmTarget -import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget -import org.jetbrains.kotlin.gradle.tasks.KotlinTest - - -plugins { - alias(libs.plugins.kotlinMultiplatform) - alias(libs.plugins.kotlinSerialization) - alias(libs.plugins.androidLibrary) - alias(libs.plugins.mavenPublishPlugin) - alias(libs.plugins.kotlinter) - id("com.powersync.plugins.sonatype") - alias(libs.plugins.mokkery) - alias(libs.plugins.kotlin.atomicfu) - id("com.powersync.plugins.sharedbuild") -} - -kotlin { - androidTarget { - publishLibraryVariants("release", "debug") - publishLibraryVariants("release", "debug") - - @OptIn(ExperimentalKotlinGradlePluginApi::class) - compilerOptions { - jvmTarget.set(JvmTarget.JVM_17) - } - } - jvm { - @OptIn(ExperimentalKotlinGradlePluginApi::class) - compilerOptions { - jvmTarget.set(JvmTarget.JVM_1_8) - // https://jakewharton.com/kotlins-jdk-release-compatibility-flag/ - freeCompilerArgs.add("-Xjdk-release=8") - } - } - - iosX64() - iosArm64() - iosSimulatorArm64() - - targets.withType { - compilations.named("main") { - compileTaskProvider { - compilerOptions.freeCompilerArgs.add("-Xexport-kdoc") - } - } - } - - explicitApi() - - applyDefaultHierarchyTemplate() - sourceSets { - all { - languageSettings { - optIn("kotlinx.cinterop.ExperimentalForeignApi") - } - } - - val commonIntegrationTest by creating { - dependsOn(commonTest.get()) - } - - commonMain.dependencies { - api(project(":core")) - implementation(libs.uuid) - implementation(libs.kotlin.stdlib) - implementation(libs.ktor.client.core) - implementation(libs.ktor.client.contentnegotiation) - implementation(libs.ktor.serialization.json) - implementation(libs.kotlinx.io) - implementation(libs.kotlinx.coroutines.core) - implementation(libs.kotlinx.datetime) - implementation(libs.stately.concurrency) - implementation(libs.configuration.annotations) - api(projects.persistence) - api(libs.kermit) - } - - androidMain { - dependencies.implementation(libs.ktor.client.okhttp) - } - - jvmMain { - dependencies { - implementation(libs.ktor.client.okhttp) - } - } - - iosMain.dependencies { - implementation(libs.ktor.client.ios) - } - - commonTest.dependencies { - implementation(libs.kotlin.test) - implementation(libs.test.coroutines) - implementation(libs.test.turbine) - implementation(libs.kermit.test) - implementation(libs.ktor.client.mock) - implementation(libs.test.turbine) - // Allows using the core test utils - implementation(project(":core")) - } - - jvmTest.get().dependsOn(commonIntegrationTest) - iosSimulatorArm64Test.orNull?.dependsOn(commonIntegrationTest) - } -} - -android { - compileOptions { - targetCompatibility = JavaVersion.VERSION_17 - } - - buildFeatures { - buildConfig = true - } - - buildTypes { - release { - buildConfigField("boolean", "DEBUG", "false") - } - debug { - buildConfigField("boolean", "DEBUG", "true") - } - } - - namespace = "com.powersync.attachments" - compileSdk = - libs.versions.android.compileSdk - .get() - .toInt() - defaultConfig { - minSdk = - libs.versions.android.minSdk - .get() - .toInt() - consumerProguardFiles("proguard-rules.pro") - } -} - -tasks.withType { - testLogging { - events("PASSED", "FAILED", "SKIPPED") - exceptionFormat = TestExceptionFormat.FULL - showStandardStreams = true - showStackTraces = true - } -} - -setupGithubRepository() diff --git a/attachments/gradle.properties b/attachments/gradle.properties deleted file mode 100644 index e69de29b..00000000 diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestUtils.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestUtils.kt deleted file mode 100644 index 4684097c..00000000 --- a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestUtils.kt +++ /dev/null @@ -1,7 +0,0 @@ -package com.powersync.attachments.testutils - -import com.powersync.DatabaseDriverFactory - -expect val factory: DatabaseDriverFactory - -expect fun cleanup(path: String) diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/UserRow.kt b/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/UserRow.kt deleted file mode 100644 index 63d550b5..00000000 --- a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/UserRow.kt +++ /dev/null @@ -1,35 +0,0 @@ -package com.powersync.attachments.testutils - -import com.powersync.db.SqlCursor -import com.powersync.db.getString -import com.powersync.db.getStringOptional -import com.powersync.db.schema.Column -import com.powersync.db.schema.Table - -data class UserRow( - val id: String, - val name: String, - val email: String, - val photoId: String?, -) { - companion object { - fun from(cursor: SqlCursor): UserRow = - UserRow( - id = cursor.getString("id"), - name = cursor.getString("name"), - email = cursor.getString("email"), - photoId = cursor.getStringOptional("photo_id"), - ) - - val table = - Table( - name = "users", - columns = - listOf( - Column.text("name"), - Column.text("email"), - Column.text("photo_id"), - ), - ) - } -} diff --git a/attachments/src/iosSimulatorArm64Test/kotlin/com/powersync/attachments/testutils/TestUtils.iosSimulatorArm64.kt b/attachments/src/iosSimulatorArm64Test/kotlin/com/powersync/attachments/testutils/TestUtils.iosSimulatorArm64.kt deleted file mode 100644 index 67a1b81a..00000000 --- a/attachments/src/iosSimulatorArm64Test/kotlin/com/powersync/attachments/testutils/TestUtils.iosSimulatorArm64.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.powersync.attachments.testutils - -import com.powersync.DatabaseDriverFactory -import kotlinx.io.files.Path -import kotlinx.io.files.SystemFileSystem - -actual val factory: DatabaseDriverFactory - get() = DatabaseDriverFactory() - -actual fun cleanup(path: String) { - val resolved = Path(path) - if (SystemFileSystem.exists(resolved)) { - SystemFileSystem.delete(resolved) - } -} diff --git a/attachments/src/jvmTest/kotlin/com/powersync/attachments/testutils/TestUtils.jvm.kt b/attachments/src/jvmTest/kotlin/com/powersync/attachments/testutils/TestUtils.jvm.kt deleted file mode 100644 index 2d558118..00000000 --- a/attachments/src/jvmTest/kotlin/com/powersync/attachments/testutils/TestUtils.jvm.kt +++ /dev/null @@ -1,11 +0,0 @@ -package com.powersync.attachments.testutils - -import com.powersync.DatabaseDriverFactory -import java.io.File - -actual val factory: DatabaseDriverFactory - get() = DatabaseDriverFactory() - -actual fun cleanup(path: String) { - File(path).delete() -} diff --git a/attachments/src/androidMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.android.kt b/core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt similarity index 84% rename from attachments/src/androidMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.android.kt rename to core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt index af7f8b76..5e78847c 100644 --- a/attachments/src/androidMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.android.kt +++ b/core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt @@ -3,6 +3,7 @@ package com.powersync.attachments.storage import android.os.Environment import com.powersync.attachments.LocalStorageAdapter +@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { actual override fun getUserStorageDirectory(): String = Environment.getDataDirectory().absolutePath } diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt similarity index 95% rename from attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt rename to core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index bd23acfb..011a8dec 100644 --- a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -1,12 +1,15 @@ -package com.powersync.attachments +package com.powersync import app.cash.turbine.turbineScope import co.touchlab.kermit.ExperimentalKermitApi -import com.powersync.PowerSyncDatabase -import com.powersync.attachments.testutils.MockedRemoteStorage -import com.powersync.attachments.testutils.TestAttachmentsQueue -import com.powersync.attachments.testutils.UserRow +import com.powersync.attachments.Attachment +import com.powersync.attachments.AttachmentState +import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.createAttachmentsTable import com.powersync.db.schema.Schema +import com.powersync.testutils.MockedRemoteStorage +import com.powersync.testutils.TestAttachmentsQueue +import com.powersync.testutils.UserRow import dev.mokkery.matcher.any import dev.mokkery.spy import dev.mokkery.verifySuspend @@ -27,7 +30,7 @@ class AttachmentsTest { private fun openDB() = PowerSyncDatabase( - factory = com.powersync.attachments.testutils.factory, + factory = com.powersync.testutils.factory, schema = Schema(UserRow.table, createAttachmentsTable("attachments")), dbFilename = "testdb", ) @@ -49,7 +52,7 @@ class AttachmentsTest { database.close() } } - com.powersync.attachments.testutils + com.powersync.testutils .cleanup("testdb") } diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt similarity index 92% rename from attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt rename to core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt index 20a39940..4e79751b 100644 --- a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/MockedRemoteStorage.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt @@ -1,4 +1,4 @@ -package com.powersync.attachments.testutils +package com.powersync.testutils import com.powersync.attachments.RemoteStorageAdapter import kotlinx.coroutines.flow.Flow diff --git a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt similarity index 95% rename from attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt rename to core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt index c1fee706..502c8fc1 100644 --- a/attachments/src/commonIntegrationTest/kotlin/com/powersync/attachments/testutils/TestAttachmentsQueue.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt @@ -1,4 +1,4 @@ -package com.powersync.attachments.testutils +package com.powersync.testutils import com.powersync.PowerSyncDatabase import com.powersync.attachments.AbstractAttachmentQueue diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/UserRow.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/UserRow.kt index caf65765..78f339f5 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/UserRow.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/UserRow.kt @@ -2,6 +2,7 @@ package com.powersync.testutils import com.powersync.db.SqlCursor import com.powersync.db.getString +import com.powersync.db.getStringOptional import com.powersync.db.schema.Column import com.powersync.db.schema.Table @@ -9,6 +10,7 @@ data class UserRow( val id: String, val name: String, val email: String, + val photo_id: String?, ) { companion object { fun from(cursor: SqlCursor): UserRow = @@ -16,8 +18,18 @@ data class UserRow( id = cursor.getString("id"), name = cursor.getString("name"), email = cursor.getString("email"), + photo_id = cursor.getStringOptional("photo_id"), ) - val table = Table(name = "users", columns = listOf(Column.text("name"), Column.text("email"))) + val table = + Table( + name = "users", + columns = + listOf( + Column.text("name"), + Column.text("email"), + Column.text("photo_id"), + ), + ) } } diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt similarity index 100% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/AbstractAttachmentQueue.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt similarity index 100% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/Attachment.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt similarity index 100% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentService.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentTable.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt similarity index 100% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/AttachmentTable.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt similarity index 100% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/LocalStorageAdapter.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorageAdapter.kt similarity index 100% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/RemoteStorageAdapter.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorageAdapter.kt diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/SyncErrorHandler.kt b/core/src/commonMain/kotlin/com/powersync/attachments/SyncErrorHandler.kt similarity index 100% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/SyncErrorHandler.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/SyncErrorHandler.kt diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt similarity index 85% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt index fe64f8dd..0d6b967e 100644 --- a/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt @@ -2,6 +2,7 @@ package com.powersync.attachments.storage import com.powersync.attachments.LocalStorageAdapter +@OptIn(ExperimentalMultiplatform::class) public expect abstract class AbstractLocalStorageAdapter() : LocalStorageAdapter { override fun getUserStorageDirectory(): String } diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt similarity index 100% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/storage/IOLocalStorageAdapter.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt diff --git a/attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt similarity index 100% rename from attachments/src/commonMain/kotlin/com.powersync.attachments/sync/SyncingService.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt diff --git a/attachments/src/iosMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.ios.kt b/core/src/iosMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.ios.kt similarity index 83% rename from attachments/src/iosMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.ios.kt rename to core/src/iosMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.ios.kt index 4d324d65..87094951 100644 --- a/attachments/src/iosMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.ios.kt +++ b/core/src/iosMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.ios.kt @@ -3,6 +3,7 @@ package com.powersync.attachments.storage import com.powersync.attachments.LocalStorageAdapter import platform.Foundation.NSHomeDirectory +@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { actual override fun getUserStorageDirectory(): String = NSHomeDirectory() } diff --git a/attachments/src/jvmMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.jvm.kt b/core/src/jvmMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.jvm.kt similarity index 82% rename from attachments/src/jvmMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.jvm.kt rename to core/src/jvmMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.jvm.kt index d7600f98..b4caa2a3 100644 --- a/attachments/src/jvmMain/kotlin/com.powersync.attachments/storage/AbstractLocalStorageAdapter.jvm.kt +++ b/core/src/jvmMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.jvm.kt @@ -2,6 +2,7 @@ package com.powersync.attachments.storage import com.powersync.attachments.LocalStorageAdapter +@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { actual override fun getUserStorageDirectory(): String = System.getProperty("user.home") } From adbacfbf9030b5fadca5307ba94769a7fd4ed1c5 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 3 Apr 2025 10:03:18 +0200 Subject: [PATCH 05/51] wip: readme --- .../kotlin/com/powersync/AttachmentsTest.kt | 6 +- .../testutils/TestAttachmentsQueue.kt | 6 +- .../attachments/AbstractAttachmentQueue.kt | 4 +- .../com/powersync/attachments/README.md | 240 ++++++++++++++++++ 4 files changed, 252 insertions(+), 4 deletions(-) create mode 100644 core/src/commonMain/kotlin/com/powersync/attachments/README.md diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 011a8dec..ff07cba5 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -194,7 +194,11 @@ class AttachmentsTest { * Creates an attachment given a flow of bytes (the file data) then assigns this to * a newly created user. */ - queue.saveFile(flowOf(ByteArray(1)), "image/jpg", "jpg") { tx, attachment -> + queue.saveFile( + data = flowOf(ByteArray(1)), + mediaType = "image/jpg", + fileExtension = "jpg", + ) { tx, attachment -> // Set the photo_id of a new user to the attachment id tx.execute( """ diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt index 502c8fc1..b314ddbe 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt @@ -11,7 +11,11 @@ internal class TestAttachmentsQueue( db: PowerSyncDatabase, remoteStorage: RemoteStorageAdapter, attachmentDirectoryName: String, -) : AbstractAttachmentQueue(db, remoteStorage, attachmentDirectoryName = attachmentDirectoryName) { +) : AbstractAttachmentQueue( + db, + remoteStorage, + attachmentDirectoryName = attachmentDirectoryName, + ) { override fun watchAttachments(): Flow> = db.watch( sql = diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt index 98aeea85..95a5364f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt @@ -17,7 +17,7 @@ import kotlinx.coroutines.sync.withLock import kotlinx.io.files.Path import kotlin.coroutines.cancellation.CancellationException import kotlin.time.Duration -import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds /** * A watched attachment record item. @@ -79,7 +79,7 @@ public abstract class AbstractAttachmentQueue( /** * Periodic interval to trigger attachment sync operations */ - private val syncInterval: Duration = 5.minutes, + private val syncInterval: Duration = 30.seconds, /** * Creates a list of subdirectories in the {attachmentDirectoryName} directory */ diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md new file mode 100644 index 00000000..eb8d19be --- /dev/null +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -0,0 +1,240 @@ +# PowerSync Attachments + +A [PowerSync](https://powersync.com) library to manage attachments in Kotlin Multiplatform apps. + +This package is included in the PowerSync Core module. + +## Usage + +An `AttachmentQueue` is used to manage and sync attachments in your app. The attachments' state are +stored in a local only attachments table. + +### Example + +In this example, the user captures photos when checklist items are completed as part of an +inspection workflow. + +The schema for the `checklist` table: + +```kotlin +import com.powersync.attachments.AbstractAttachmentQueue +import com.powersync.attachments.createAttachmentsTable +import com.powersync.db.schema.Schema +import com.powersync.db.schema.Table + +val checklists = Table( + name = "checklists", + columns = + listOf( + Column.text("description"), + Column.integer("completed"), + Column.text("photo_id"), + ), +) + +val schema = Schema( + UserRow.table, + // Includes the table which stores attachment states + createAttachmentsTable("attachments") +) +``` + +The `createAttachmentsTable` function defines the local only attachment state storage table. + +An attachments table definition can be created with the following options. + +| Option | Description | Default | +|---------------------|---------------------------------------------------------------------------------|-------------------------------| +| `name` | The name of the table | `attachments` | +| `additionalColumns` | An array of addition `Column` objects added to the default columns in the table | See below for default columns | + +The default columns in `AttachmentTable`: + +| Column Name | Type | Description | +|--------------|-----------|-------------------------------------------------------------------| +| `id` | `TEXT` | The ID of the attachment record | +| `filename` | `TEXT` | The filename of the attachment | +| `media_type` | `TEXT` | The media type of the attachment | +| `state` | `INTEGER` | The state of the attachment, one of `AttachmentState` enum values | +| `timestamp` | `INTEGER` | The timestamp of last update to the attachment record | +| `size` | `INTEGER` | The size of the attachment in bytes | + +### Steps to implement + +1. Create a new `AttachmentQueue` class that extends `AbstractAttachmentQueue` from + `com.powersync.attachments`. + +```kotlin + +class AttachmentsQueue( + db: PowerSyncDatabase, + remoteStorage: RemoteStorageAdapter, + attachmentDirectoryName: String, +) : AbstractAttachmentQueue( + db, + remoteStorage, + attachmentDirectoryName = attachmentDirectoryName, + // See the class definition for more options +) { + + // An example implementation + override fun watchAttachments(): Flow> = + db.watch( + sql = + """ + SELECT + photo_id + FROM + checklists + WHERE + photo_id IS NOT NULL + """, + ) { + WatchedAttachmentItem(id = it.getString("photo_id"), fileExtension = "jpg") + } +} +``` + +2. Implement `watchAttachments`, which returns a `Flow` of `WatchedAttachmentItem`. + The `WatchedAttachmentItem`s represent the attachments which should be present in the + application. We recommend using `PowerSync`'s `watch` query as shown above. In this example we + provide the `fileExtension` for all photos. This information could also be + obtained from the query if necessary. + +3. To instantiate an `AttachmentQueue`, one needs to provide an instance of + `PowerSyncDatabase` from PowerSync and an instance of `RemoteStorageAdapter`. The remote storage + is responsible for connecting to the attachments backend. See the `RemoteStorageAdapter` + interface + definition [here](https://github.com/powersync-ja/powersync-kotlin/blob/main/core/src/commonMain/kotlin/com.powersync/attachments/RemoteStorageAdapter.ts). + + +4. Implement a `RemoteStorageAdapter` which interfaces with a remote storage provider. This will be + used for downloading, uploading and deleting attachments. + +```kotlin +val remote = object : RemoteStorageAdapter() { + override suspend fun uploadFile( + filename: String, + file: Flow, + mediaType: String? + ) { + TODO("Make a request to the backend") + } + + override suspend fun downloadFile(filename: String): Flow { + TODO("Make a request to the backend") + } + + override suspend fun deleteFile(filename: String) { + TODO("Make a request to the backend") + } + +} + +``` + +5. Instantiate a new `AttachmentQueue` and call `start()` to start syncing attachments. + +```kotlin + val queue = + AttachmentsQueue( + db = database, + remoteStorage = remote, + ) + +queue.start() +``` + +7. Finally, to create an attachment and add it to the queue, call `saveFile()`. This method will + save the file to the local storage, create an attachment record which queues the file for upload + to the remote storage and allows assigning the newly created attachment ID to a checklist item. + +```kotlin +queue.saveFile( + // The attachment's data flow + data = flowOf(ByteArray(1)), + mediaType = "image/jpg", + fileExtension = "jpg", +) { tx, attachment -> + // Set the photo_id of a checklist to the attachment id + tx.execute( + """ + UPDATE + checklists + SET + photo_id = ? + WHERE + id = ? + """, + listOf(attachment.id, checklistId), + ) +} +``` + +# Implementation details + +## Attachment State + +The `AttachmentQueue` class manages attachments in your app by tracking their state. + +The state of an attachment can be one of the following: + +| State | Description | +|-------------------|-------------------------------------------------------------------------------| +| `QUEUED_UPLOAD` | The attachment has been queued for upload to the cloud storage | +| `QUEUED_DOWNLOAD` | The attachment has been queued for download from the cloud storage | +| `SYNCED` | The attachment has been synced | +| `ARCHIVED` | The attachment has been orphaned, i.e. the associated record has been deleted | + +## Syncing attachments + +The `AttachmentQueue` sets a watched query on the `attachments` table, for record in the +`QUEUED_UPLOAD` and `QUEUED_DOWNLOAD` state. An event loop triggers calls to the remote storage for +these operations. + +In addition to watching for changes, the `AttachmentQueue` also triggers a sync periodically. +This will retry any failed uploads/downloads, in particular after the app was offline. + +By default, this is every 30 seconds, but can be configured by setting `syncInterval` in the +`AttachmentQueue` constructor options, or disabled by setting the interval to `0`. + +### Uploading + +- An `AttachmentRecord` is created or updated with a state of `QUEUED_UPLOAD`. +- The `AttachmentQueue` picks this up and upon successful upload to the remote storage, sets the + state to + `SYNCED`. +- If the upload is not successful, the record remains in `QUEUED_UPLOAD` state and uploading will be + retried when syncing triggers again. Retries can be stopped by providing an `errorHandler`. + +### Downloading + +- An `AttachmentRecord` is created or updated with `QUEUED_DOWNLOAD` state. +- The watched query from `watchAttachments` adds the attachment `id` into a queue of IDs to download + and triggers the download process +- The watched query from `watchAttachments` adds the attachment `id` into a queue of IDs to download + and triggers the download process +- If the photo is not on the device, it is downloaded from cloud storage. +- Writes file to the user's local storage. +- If this is successful, update the `AttachmentRecord` state to `SYNCED`. +- If any of these fail, the download is retried in the next sync trigger. + +### Deleting attachments + +When an attachment is deleted by a user action or cache expiration: + +- Related `AttachmentRecord` is removed from attachments table. +- Local file (if exists) is deleted. +- File on cloud storage is deleted. + +### Expire Cache + +When PowerSync removes a record, as a result of coming back online or conflict resolution for +instance: + +- Any associated `AttachmentRecord` is orphaned. +- On the next sync trigger, the `AttachmentQueue` sets all records that are orphaned to `ARCHIVED` + state. +- By default, the `AttachmentQueue` only keeps the last `100` attachment records and then expires + the rest. +- This can be configured by setting `cacheLimit` in the `AttachmentQueue` constructor options. From d18e4d01c4415076e8803845c56f64af0c07fad5 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 3 Apr 2025 14:59:33 +0200 Subject: [PATCH 06/51] README updates --- .../kotlin/com/powersync/AttachmentsTest.kt | 134 ++++++++++++++++-- .../testutils/TestAttachmentsQueue.kt | 9 +- .../attachments/AbstractAttachmentQueue.kt | 116 ++++++++++++--- .../com/powersync/attachments/Attachment.kt | 6 + .../attachments/AttachmentService.kt | 30 +++- .../powersync/attachments/AttachmentTable.kt | 1 + .../attachments/LocalStorageAdapter.kt | 13 +- .../com/powersync/attachments/README.md | 39 +++-- .../storage/AbstractLocalStorageAdapter.kt | 2 +- .../storage/IOLocalStorageAdapter.kt | 68 +++++---- .../attachments/sync/SyncingService.kt | 29 ++-- 11 files changed, 358 insertions(+), 89 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index ff07cba5..25adf52d 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -23,6 +23,7 @@ import kotlin.test.assertEquals import kotlin.test.assertNotNull import kotlin.test.assertNull import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.minutes @OptIn(ExperimentalKermitApi::class) class AttachmentsTest { @@ -56,24 +57,21 @@ class AttachmentsTest { .cleanup("testdb") } - @Test - fun testLinksPowerSync() = - runTest { - database.get("SELECT powersync_rs_version();") { it.getString(0)!! } - } - @Test fun testAttachmentDownload() = - runTest { - turbineScope { + runTest(timeout = 5.minutes) { + turbineScope(timeout = 5.minutes) { val remote = spy(MockedRemoteStorage()) val queue = TestAttachmentsQueue( db = database, remoteStorage = remote, - // For these tests (jvm/ios) this should be fine for now - attachmentDirectoryName = "/tmp", + /** + * Sets the cache limit to zero for this test. Archived records will + * immediately be deleted. + */ + archivedCacheLimit = 0, ) queue.start() @@ -173,8 +171,11 @@ class AttachmentsTest { TestAttachmentsQueue( db = database, remoteStorage = remote, - // For these tests (jvm/ios) this should be fine for now - attachmentDirectoryName = "/tmp", + /** + * Sets the cache limit to zero for this test. Archived records will + * immediately be deleted. + */ + archivedCacheLimit = 0, ) queue.start() @@ -258,6 +259,115 @@ class AttachmentsTest { // The file should have been deleted from storage assertEquals(expected = false, actual = queue.localStorage.fileExists(localUri)) + attachmentQuery.cancel() + queue.close() + } + } + + @Test + fun testAttachmentCachedDownload() = + runTest { + turbineScope { + val remote = spy(MockedRemoteStorage()) + + val queue = + TestAttachmentsQueue( + db = database, + remoteStorage = remote, + /** + * Keep some items in the cache + */ + archivedCacheLimit = 10, + ) + + queue.start() + + // Monitor the attachments table for testing + val attachmentQuery = + database + .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } + .testIn(this) + + val result = attachmentQuery.awaitItem() + + // There should not be any attachment records here + assertEquals(expected = 0, actual = result.size) + + // Create a user with a photo_id specified. + // This code did not save an attachment before assigning a photo_id. + // This is equivalent to requiring an attachment download + database.execute( + """ + INSERT INTO + users (id, name, email, photo_id) + VALUES + (uuid(), "steven", "steven@journeyapps.com", uuid()) + """, + ) + + var attachmentRecord = attachmentQuery.awaitItem().first() + assertNotNull( + attachmentRecord, + """ + An attachment record should be created after creating a user with a photo_id + " + """.trimIndent(), + ) + + /** + * The timing of the watched query resolving might differ slightly. + * We might get a watched query result where the attachment is QUEUED_DOWNLOAD + * or we could get the result once it has been DOWNLOADED. + * We should assert that the download happens eventually. + */ + + if (attachmentRecord.state == AttachmentState.QUEUED_DOWNLOAD.ordinal) { + // Wait for the download to be triggered + attachmentRecord = attachmentQuery.awaitItem().first() + } + + assertEquals(expected = AttachmentState.SYNCED.ordinal, attachmentRecord.state) + + // A download should have been attempted for this file + verifySuspend { remote.downloadFile(attachmentRecord.filename) } + + // A file should now exist + val localUri = attachmentRecord.localUri!! + assertTrue { queue.localStorage.fileExists(localUri) } + + // Now clear the user's photo_id. The attachment should be archived + database.execute( + """ + UPDATE + users + SET + photo_id = NULL + """, + ) + + attachmentRecord = attachmentQuery.awaitItem().first() + assertEquals( + expected = AttachmentState.ARCHIVED.ordinal, + actual = attachmentRecord.state, + ) + + // Now if we set the photo_id, the archived record should be restored + database.execute( + """ + UPDATE + users + SET + photo_id = ? + """, + listOf(attachmentRecord.id), + ) + + attachmentRecord = attachmentQuery.awaitItem().first() + assertEquals( + expected = AttachmentState.SYNCED.ordinal, + actual = attachmentRecord.state, + ) + attachmentQuery.cancel() queue.close() } diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt index b314ddbe..4b407271 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt @@ -10,11 +10,11 @@ import kotlinx.coroutines.flow.Flow internal class TestAttachmentsQueue( db: PowerSyncDatabase, remoteStorage: RemoteStorageAdapter, - attachmentDirectoryName: String, + archivedCacheLimit: Long, ) : AbstractAttachmentQueue( db, remoteStorage, - attachmentDirectoryName = attachmentDirectoryName, + archivedCacheLimit = archivedCacheLimit, ) { override fun watchAttachments(): Flow> = db.watch( @@ -30,4 +30,9 @@ internal class TestAttachmentsQueue( ) { WatchedAttachmentItem(id = it.getString("photo_id"), fileExtension = "jpg") } + + /** + * For tests this uses a temporary directory. On iOS it uses the user storage directory + */ + override fun getStorageDirectory(): String = getTempDir() ?: super.getStorageDirectory() } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt index 95a5364f..414df41b 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt @@ -3,6 +3,7 @@ package com.powersync.attachments import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase import com.powersync.PowerSyncException +import com.powersync.attachments.storage.IOLocalStorageAdapter import com.powersync.attachments.sync.SyncingService import com.powersync.db.internal.ConnectionContext import com.powersync.db.runWrappedSuspending @@ -64,7 +65,8 @@ public abstract class AbstractAttachmentQueue( */ public val localStorage: LocalStorageAdapter = IOLocalStorageAdapter(), /** - * Directory where attachment files will be written to disk + * Directory name where attachment files will be written to disk. + * This will be created under the directory returned from [getStorageDirectory] */ private val attachmentDirectoryName: String = DEFAULT_ATTACHMENTS_DIRECTORY_NAME, /** @@ -80,10 +82,20 @@ public abstract class AbstractAttachmentQueue( * Periodic interval to trigger attachment sync operations */ private val syncInterval: Duration = 30.seconds, + /** + * Archived attachments can be used as a cache which can be restored if an attachment id + * reappears after being removed. This parameter defines how many archived records are retained. + * Records are deleted once the number of items exceeds this value. + */ + private val archivedCacheLimit: Long = 100, /** * Creates a list of subdirectories in the {attachmentDirectoryName} directory */ private val subdirectories: List? = null, + /** + * Should attachments be downloaded + */ + private val downloadAttachments: Boolean = true, /** * Logging interface used for all log operations */ @@ -101,7 +113,12 @@ public abstract class AbstractAttachmentQueue( * - Create new attachment records for upload/download */ public val attachmentsService: AttachmentService = - AttachmentService(db, attachmentsQueueTableName, logger) + AttachmentService( + db, + attachmentsQueueTableName, + logger, + maxArchivedCount = archivedCacheLimit, + ) /** * Syncing service for this attachment queue. @@ -151,20 +168,27 @@ public abstract class AbstractAttachmentQueue( // Listen for connectivity changes syncStatusJob = scope.launch { - scope.launch { - db.currentStatus.asFlow().collect { status -> - if (status.connected) { - syncingService.triggerSync() + val statusJob = + scope.launch { + var previousConnected = db.currentStatus.connected + db.currentStatus.asFlow().collect { status -> + if (!previousConnected && status.connected) { + syncingService.triggerSync() + } + previousConnected = status.connected } } - } - scope.launch { - // Watch local attachment relationships and sync the attachment records - watchAttachments().collect { items -> - processWatchedAttachments(items) + val watchJob = + scope.launch { + // Watch local attachment relationships and sync the attachment records + watchAttachments().collect { items -> + processWatchedAttachments(items) + } } - } + + statusJob.join() + watchJob.join() } } } @@ -217,7 +241,7 @@ public abstract class AbstractAttachmentQueue( * This method can be overriden for custom behaviour. */ @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun resolveNewAttachmentFilename( + public open suspend fun resolveNewAttachmentFilename( attachmentId: String, fileExtension: String?, ): String = "$attachmentId.$fileExtension" @@ -237,8 +261,12 @@ public abstract class AbstractAttachmentQueue( * This method can be overriden for custom behaviour. */ @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun processWatchedAttachments(items: List): Unit = + public open suspend fun processWatchedAttachments(items: List): Unit = runWrappedSuspending { + /** + * Need to get all the attachments which are tracked in the DB. + * We might need to restore an archived attachment. + */ val currentAttachments = attachmentsService.getAttachments() val attachmentUpdates = mutableListOf() @@ -246,6 +274,9 @@ public abstract class AbstractAttachmentQueue( val existingQueueItem = currentAttachments.find { it.id == item.id } if (existingQueueItem == null) { + if (!downloadAttachments) { + continue + } // This item should be added to the queue // This item is assumed to be coming from an upstream sync // Locally created new items should be persisted using [saveFile] before @@ -263,10 +294,36 @@ public abstract class AbstractAttachmentQueue( state = AttachmentState.QUEUED_DOWNLOAD.ordinal, ), ) + } else if + (existingQueueItem.state == AttachmentState.ARCHIVED.ordinal) { + // The attachment is present again. Need to queue it for sync. + // We might be able to optimize this in future + if (existingQueueItem.hasSynced == 1) { + // No remote action required, we can restore the record (avoids deletion) + attachmentUpdates.add( + existingQueueItem.copy(state = AttachmentState.SYNCED.ordinal), + ) + } else { + /** + * The localURI should be set if the record was meant to be downloaded + * and has been synced. If it's missing and hasSynced is false then + * it must be an upload operation + */ + attachmentUpdates.add( + existingQueueItem.copy( + state = + if (existingQueueItem.localUri == null) { + AttachmentState.QUEUED_DOWNLOAD.ordinal + } else { + AttachmentState.QUEUED_UPLOAD.ordinal + }, + ), + ) + } } } - // Remove any items not specified in the items + // Archive any items not specified in the watched items currentAttachments .filter { null == items.find { update -> update.id == it.id } } .forEach { @@ -289,7 +346,7 @@ public abstract class AbstractAttachmentQueue( * This method can be overriden for custom behaviour. */ @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun saveFile( + public open suspend fun saveFile( data: Flow, mediaType: String, fileExtension: String?, @@ -335,7 +392,7 @@ public abstract class AbstractAttachmentQueue( * This method can be overriden for custom behaviour. */ @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun deleteFile(attachmentId: String): Attachment = + public open suspend fun deleteFile(attachmentId: String): Attachment = runWrappedSuspending { val attachment = attachmentsService.getAttachment(attachmentId) @@ -350,13 +407,13 @@ public abstract class AbstractAttachmentQueue( * Returns the local file path for the given filename, used to store in the database. * Example: filename: "attachment-1.jpg" returns "attachments/attachment-1.jpg" */ - public fun getLocalFilePathSuffix(filename: String): String = Path(attachmentDirectoryName, filename).toString() + public open fun getLocalFilePathSuffix(filename: String): String = Path(attachmentDirectoryName, filename).toString() /** * Returns the directory where attachments are stored on the device, used to make dir * Example: "/data/user/0/com.yourdomain.app/files/attachments/" */ - public fun getStorageDirectory(): String { + public open fun getStorageDirectory(): String { val userStorageDirectory = localStorage.getUserStorageDirectory() return Path(userStorageDirectory, attachmentDirectoryName).toString() } @@ -365,8 +422,27 @@ public abstract class AbstractAttachmentQueue( * Return users storage directory with the attachmentPath use to load the file. * Example: filePath: "attachments/attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg" */ - public fun getLocalUri(filename: String): String { + public open fun getLocalUri(filename: String): String { val storageDirectory = getStorageDirectory() return Path(storageDirectory, filename).toString() } + + /** + * Removes all archived items + */ + public suspend fun expireCache() { + var done: Boolean + do { + done = syncingService.deleteArchivedAttachments() + } while (!done) + } + + /** + * Clears the attachment queue and deletes all attachment files + */ + public suspend fun clearQueue() { + attachmentsService.clearQueue() + // Remove the attachments directory + localStorage.rmDir(getStorageDirectory()) + } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt index f9cf0017..0810ceec 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt @@ -28,6 +28,11 @@ public data class Attachment( val localUri: String? = null, val mediaType: String? = null, val size: Long? = null, + /** + * Specifies if the attachment has been synced locally before. This is particularly useful + * for restoring archived attachments in edge cases. + */ + val hasSynced: Int = 0, ) { public companion object { public fun fromCursor(cursor: SqlCursor): Attachment = @@ -39,6 +44,7 @@ public data class Attachment( mediaType = cursor.getStringOptional(name = "media_type"), size = cursor.getLongOptional("size"), state = cursor.getLong("state").toInt(), + hasSynced = cursor.getLong("has_synced").toInt(), ) } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt index 5a2868a2..74a84f7c 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt @@ -16,6 +16,7 @@ public class AttachmentService( private val db: PowerSyncDatabase, private val tableName: String, private val logger: Logger, + private val maxArchivedCount: Long, ) { /** * Table used for storing attachments in the attachment queue. @@ -158,13 +159,30 @@ public class AttachmentService( /** * Delete attachments which have been archived + * @returns true if all items have been deleted. Returns false if there might be more archived + * items remaining. */ - public suspend fun deleteArchivedAttachments(callback: suspend (attachments: List) -> Unit) { + public suspend fun deleteArchivedAttachments(callback: suspend (attachments: List) -> Unit): Boolean { // First fetch the attachments in order to allow other cleanup + val limit = 1000 val attachments = db.getAll( - "SELECT * FROM $table WHERE state = ?", - listOf(AttachmentState.ARCHIVED.ordinal), + """ + SELECT + * + FROM + $table + WHERE + state = ? + ORDER BY + timestamp DESC + LIMIT ? OFFSET ? + """, + listOf( + AttachmentState.ARCHIVED.ordinal, + limit, + maxArchivedCount, + ), ) { Attachment.fromCursor(it) } callback(attachments) db.execute( @@ -173,6 +191,7 @@ public class AttachmentService( Json.encodeToString(attachments.map { it.id }), ), ) + return attachments.size < limit } /** @@ -190,9 +209,9 @@ public class AttachmentService( context.execute( """ INSERT OR REPLACE INTO - $table (id, timestamp, filename, local_uri, media_type, size, state) + $table (id, timestamp, filename, local_uri, media_type, size, state, has_synced) VALUES - (?, ?, ?, ?, ?, ?, ?) + (?, ?, ?, ?, ?, ?, ?, ?) """, listOf( updatedRecord.id, @@ -202,6 +221,7 @@ public class AttachmentService( updatedRecord.mediaType, updatedRecord.size, updatedRecord.state, + updatedRecord.hasSynced, ), ) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt index 78adbbbb..978cf81d 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt @@ -21,6 +21,7 @@ public fun createAttachmentsTable( Column("size", ColumnType.INTEGER), Column("media_type", ColumnType.TEXT), Column("state", ColumnType.INTEGER), + Column("has_synced", ColumnType.INTEGER), ).plus(additionalColumns ?: emptyList()), localOnly = true, ) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt index dcc66f79..c7eb53a1 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt @@ -1,6 +1,8 @@ package com.powersync.attachments +import com.powersync.PowerSyncException import kotlinx.coroutines.flow.Flow +import kotlin.coroutines.cancellation.CancellationException /** * Storage adapter for local storage @@ -10,22 +12,31 @@ public interface LocalStorageAdapter { * Saves a source of data bytes to a path. * @returns the bytesize of the file */ + @Throws(PowerSyncException::class, CancellationException::class) public suspend fun saveFile( filePath: String, data: Flow, ): Long + @Throws(PowerSyncException::class, CancellationException::class) public suspend fun readFile( filePath: String, mediaType: String? = null, ): Flow + @Throws(PowerSyncException::class, CancellationException::class) public suspend fun deleteFile(filePath: String): Unit + @Throws(PowerSyncException::class, CancellationException::class) public suspend fun fileExists(filePath: String): Boolean - public suspend fun makeDir(filePath: String): Unit + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun makeDir(path: String): Unit + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun rmDir(path: String): Unit + + @Throws(PowerSyncException::class, CancellationException::class) public suspend fun copyFile( sourcePath: String, targetPath: String, diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index eb8d19be..8badd1a4 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -9,6 +9,14 @@ This package is included in the PowerSync Core module. An `AttachmentQueue` is used to manage and sync attachments in your app. The attachments' state are stored in a local only attachments table. +### Key Assumptions + +- Attachments are immutable. +- Each attachment should be identifiable by a unique ID. +- Relational data should contain a foreign key column that references the attachment ID. +- Relational data should reflect the holistic state of attachments at any given time. An existing + local attachment will deleted locally if no relational data references it. + ### Example In this example, the user captures photos when checklist items are completed as part of an @@ -50,14 +58,15 @@ An attachments table definition can be created with the following options. The default columns in `AttachmentTable`: -| Column Name | Type | Description | -|--------------|-----------|-------------------------------------------------------------------| -| `id` | `TEXT` | The ID of the attachment record | -| `filename` | `TEXT` | The filename of the attachment | -| `media_type` | `TEXT` | The media type of the attachment | -| `state` | `INTEGER` | The state of the attachment, one of `AttachmentState` enum values | -| `timestamp` | `INTEGER` | The timestamp of last update to the attachment record | -| `size` | `INTEGER` | The size of the attachment in bytes | +| Column Name | Type | Description | +|--------------|-----------|--------------------------------------------------------------------------------------------------------------------| +| `id` | `TEXT` | The ID of the attachment record | +| `filename` | `TEXT` | The filename of the attachment | +| `media_type` | `TEXT` | The media type of the attachment | +| `state` | `INTEGER` | The state of the attachment, one of `AttachmentState` enum values | +| `timestamp` | `INTEGER` | The timestamp of last update to the attachment record | +| `size` | `INTEGER` | The size of the attachment in bytes | +| `has_synced` | `INTEGER` | Internal tracker which tracks if the attachment has ever been synced. This is used for caching/archiving purposes. | ### Steps to implement @@ -151,11 +160,15 @@ queue.start() ```kotlin queue.saveFile( - // The attachment's data flow + // The attachment's data flow, this is just an example data = flowOf(ByteArray(1)), mediaType = "image/jpg", fileExtension = "jpg", ) { tx, attachment -> + /** + * This lambda is invoked in the same transaction which creates the attachment record. + * Assignments of the newly created photo_id should be done in the same transaction for maximum efficiency. + */ // Set the photo_id of a checklist to the attachment id tx.execute( """ @@ -182,6 +195,7 @@ The state of an attachment can be one of the following: | State | Description | |-------------------|-------------------------------------------------------------------------------| | `QUEUED_UPLOAD` | The attachment has been queued for upload to the cloud storage | +| `QUEUED_DELETE` | The attachment has been queued for delete in the cloud storage (and locally) | | `QUEUED_DOWNLOAD` | The attachment has been queued for download from the cloud storage | | `SYNCED` | The attachment has been synced | | `ARCHIVED` | The attachment has been orphaned, i.e. the associated record has been deleted | @@ -189,7 +203,8 @@ The state of an attachment can be one of the following: ## Syncing attachments The `AttachmentQueue` sets a watched query on the `attachments` table, for record in the -`QUEUED_UPLOAD` and `QUEUED_DOWNLOAD` state. An event loop triggers calls to the remote storage for +`QUEUED_UPLOAD`, `QUEUED_DELETE` and `QUEUED_DOWNLOAD` state. An event loop triggers calls to the +remote storage for these operations. In addition to watching for changes, the `AttachmentQueue` also triggers a sync periodically. @@ -237,4 +252,6 @@ instance: state. - By default, the `AttachmentQueue` only keeps the last `100` attachment records and then expires the rest. -- This can be configured by setting `cacheLimit` in the `AttachmentQueue` constructor options. +- In some cases these records (attachment ids) might be restored. An archived attachment will be + restored if it is still in the cache. This can be configured by setting `cacheLimit` in the + `AttachmentQueue` constructor options. diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt index 0d6b967e..07cf77c1 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt @@ -2,7 +2,7 @@ package com.powersync.attachments.storage import com.powersync.attachments.LocalStorageAdapter -@OptIn(ExperimentalMultiplatform::class) +@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") public expect abstract class AbstractLocalStorageAdapter() : LocalStorageAdapter { override fun getUserStorageDirectory(): String } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt index 382ed390..c8e121ce 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt @@ -1,6 +1,7 @@ -package com.powersync.attachments +package com.powersync.attachments.storage import com.powersync.attachments.storage.AbstractLocalStorageAdapter +import com.powersync.db.runWrappedSuspending import io.ktor.utils.io.core.remaining import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO @@ -22,21 +23,23 @@ public class IOLocalStorageAdapter : AbstractLocalStorageAdapter() { filePath: String, data: Flow, ): Long = - withContext(Dispatchers.IO) { - var totalSize = 0L - SystemFileSystem.sink(Path(filePath)).use { sink -> - // Copy to a buffer in order to write - Buffer().use { buffer -> - data.collect { chunk -> - // Copy into a buffer in order to sink the chunk - buffer.write(chunk, 0) - val chunkSize = chunk.size.toLong() - totalSize += chunkSize - sink.write(buffer, chunkSize) + runWrappedSuspending { + withContext(Dispatchers.IO) { + var totalSize = 0L + SystemFileSystem.sink(Path(filePath)).use { sink -> + // Copy to a buffer in order to write + Buffer().use { buffer -> + data.collect { chunk -> + // Copy into a buffer in order to sink the chunk + buffer.write(chunk, 0) + val chunkSize = chunk.size.toLong() + totalSize += chunkSize + sink.write(buffer, chunkSize) + } } + sink.flush() + return@withContext totalSize } - sink.flush() - return@withContext totalSize } } @@ -64,28 +67,43 @@ public class IOLocalStorageAdapter : AbstractLocalStorageAdapter() { } public override suspend fun deleteFile(filePath: String): Unit = - withContext(Dispatchers.IO) { - SystemFileSystem.delete(Path(filePath)) + runWrappedSuspending { + withContext(Dispatchers.IO) { + SystemFileSystem.delete(Path(filePath)) + } } public override suspend fun fileExists(filePath: String): Boolean = - withContext(Dispatchers.IO) { - SystemFileSystem.exists(Path(filePath)) + runWrappedSuspending { + withContext(Dispatchers.IO) { + SystemFileSystem.exists(Path(filePath)) + } + } + + public override suspend fun makeDir(path: String): Unit = + runWrappedSuspending { + withContext(Dispatchers.IO) { + SystemFileSystem.createDirectories(Path(path)) + } } - public override suspend fun makeDir(filePath: String): Unit = - withContext(Dispatchers.IO) { - SystemFileSystem.createDirectories(Path(filePath)) + public override suspend fun rmDir(path: String): Unit = + runWrappedSuspending { + withContext(Dispatchers.IO) { + SystemFileSystem.delete(Path(path)) + } } public override suspend fun copyFile( sourcePath: String, targetPath: String, ): Unit = - withContext(Dispatchers.IO) { - SystemFileSystem.source(Path(sourcePath)).use { source -> - SystemFileSystem.sink(Path(targetPath)).use { sink -> - source.buffered().transferTo(sink.buffered()) + runWrappedSuspending { + withContext(Dispatchers.IO) { + SystemFileSystem.source(Path(sourcePath)).use { source -> + SystemFileSystem.sink(Path(targetPath)).use { sink -> + source.buffered().transferTo(sink.buffered()) + } } } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index 050fa8ef..866161d2 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -71,17 +71,7 @@ internal class SyncingService( handleSync(attachments) // Cleanup archived attachments - attachmentsService.deleteArchivedAttachments { pendingDelete -> - for (attachment in pendingDelete) { - if (attachment.localUri == null) { - continue - } - if (!localStorage.fileExists(attachment.localUri)) { - continue - } - localStorage.deleteFile(attachment.localUri) - } - } + deleteArchivedAttachments() } catch (ex: Exception) { if (ex is CancellationException) { throw ex @@ -119,10 +109,24 @@ internal class SyncingService( suspend fun close(): Unit = mutex.withLock { periodicSyncTrigger?.cancel() + periodicSyncTrigger?.join() syncJob.cancel() syncJob.join() } + suspend fun deleteArchivedAttachments() = + attachmentsService.deleteArchivedAttachments { pendingDelete -> + for (attachment in pendingDelete) { + if (attachment.localUri == null) { + continue + } + if (!localStorage.fileExists(attachment.localUri)) { + continue + } + localStorage.deleteFile(attachment.localUri) + } + } + /** * Handle downloading, uploading or deleting of attachments */ @@ -174,7 +178,7 @@ internal class SyncingService( mediaType = attachment.mediaType ?: "", ) logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") - return attachment.copy(state = AttachmentState.SYNCED.ordinal) + return attachment.copy(state = AttachmentState.SYNCED.ordinal, hasSynced = 1) } catch (e: Exception) { logger.e("Upload attachment error for attachment $attachment: ${e.message}") if (errorHandler != null) { @@ -210,6 +214,7 @@ internal class SyncingService( return attachment.copy( localUri = attachmentPath, state = AttachmentState.SYNCED.ordinal, + hasSynced = 1, ) } catch (e: Exception) { if (errorHandler != null) { From 9a9955d0bc6b853eda5abdc1adf390c97d0975d9 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 3 Apr 2025 15:59:27 +0200 Subject: [PATCH 07/51] improve storage adapters --- .../kotlin/com/powersync/AttachmentsTest.kt | 7 +++---- .../com/powersync/testutils/MockedRemoteStorage.kt | 10 +++++----- .../kotlin/com/powersync/attachments/README.md | 11 +++++------ .../com/powersync/attachments/RemoteStorageAdapter.kt | 9 ++++----- .../com/powersync/attachments/sync/SyncingService.kt | 7 +++---- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 25adf52d..6d230c20 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -123,7 +123,7 @@ class AttachmentsTest { assertEquals(expected = AttachmentState.SYNCED.ordinal, attachmentRecord.state) // A download should have been attempted for this file - verifySuspend { remote.downloadFile(attachmentRecord.filename) } + verifySuspend { remote.downloadFile(attachmentRecord) } // A file should now exist val localUri = attachmentRecord.localUri!! @@ -228,9 +228,8 @@ class AttachmentsTest { // A download should have been attempted for this file verifySuspend { remote.uploadFile( - attachmentRecord.filename, any(), - attachmentRecord.mediaType, + attachmentRecord, ) } @@ -329,7 +328,7 @@ class AttachmentsTest { assertEquals(expected = AttachmentState.SYNCED.ordinal, attachmentRecord.state) // A download should have been attempted for this file - verifySuspend { remote.downloadFile(attachmentRecord.filename) } + verifySuspend { remote.downloadFile(attachmentRecord) } // A file should now exist val localUri = attachmentRecord.localUri!! diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt index 4e79751b..6d987509 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt @@ -1,24 +1,24 @@ package com.powersync.testutils +import com.powersync.attachments.Attachment import com.powersync.attachments.RemoteStorageAdapter import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow class MockedRemoteStorage : RemoteStorageAdapter { override suspend fun uploadFile( - filename: String, - file: Flow, - mediaType: String?, + fileData: Flow, + attachment: Attachment, ) { // No op } - override suspend fun downloadFile(filename: String): Flow = + override suspend fun downloadFile(attachment: Attachment): Flow = flow { emit(ByteArray(1)) } - override suspend fun deleteFile(filename: String) { + override suspend fun deleteFile(attachment: Attachment) { // No op } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index 8badd1a4..cdc6085b 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -11,8 +11,8 @@ stored in a local only attachments table. ### Key Assumptions -- Attachments are immutable. - Each attachment should be identifiable by a unique ID. +- Attachments are immutable. - Relational data should contain a foreign key column that references the attachment ID. - Relational data should reflect the holistic state of attachments at any given time. An existing local attachment will deleted locally if no relational data references it. @@ -123,18 +123,17 @@ class AttachmentsQueue( ```kotlin val remote = object : RemoteStorageAdapter() { override suspend fun uploadFile( - filename: String, - file: Flow, - mediaType: String? + fileData: Flow, + attachment: Attachment, ) { TODO("Make a request to the backend") } - override suspend fun downloadFile(filename: String): Flow { + override suspend fun downloadFile(attachment: Attachment): Flow { TODO("Make a request to the backend") } - override suspend fun deleteFile(filename: String) { + override suspend fun deleteFile(attachment: Attachment) { TODO("Make a request to the backend") } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorageAdapter.kt index 8d21e1e3..54c3b9a8 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorageAdapter.kt @@ -10,18 +10,17 @@ public interface RemoteStorageAdapter { * Upload a file to remote storage */ public suspend fun uploadFile( - filename: String, - file: Flow, - mediaType: String?, + fileData: Flow, + attachment: Attachment, ): Unit /** * Download a file from remote storage */ - public suspend fun downloadFile(filename: String): Flow + public suspend fun downloadFile(attachment: Attachment): Flow /** * Delete a file from remote storage */ - public suspend fun deleteFile(filename: String) + public suspend fun deleteFile(attachment: Attachment) } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index 866161d2..c79f8505 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -173,9 +173,8 @@ internal class SyncingService( } remoteStorage.uploadFile( - attachment.filename, localStorage.readFile(attachment.localUri), - mediaType = attachment.mediaType ?: "", + attachment, ) logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") return attachment.copy(state = AttachmentState.SYNCED.ordinal, hasSynced = 1) @@ -206,7 +205,7 @@ internal class SyncingService( val attachmentPath = getLocalUri(attachment.filename) try { - val fileFlow = remoteStorage.downloadFile(attachment.filename) + val fileFlow = remoteStorage.downloadFile(attachment) localStorage.saveFile(attachmentPath, fileFlow) logger.i("Downloaded file \"${attachment.id}\"") @@ -236,7 +235,7 @@ internal class SyncingService( */ private suspend fun deleteAttachment(attachment: Attachment): Attachment { try { - remoteStorage.deleteFile(attachment.filename) + remoteStorage.deleteFile(attachment) if (attachment.localUri != null) { localStorage.deleteFile(attachment.localUri) } From 6172c08d5ccdc3d7b88924887d04d1b068e4b04d Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 3 Apr 2025 16:17:25 +0200 Subject: [PATCH 08/51] cleanup --- .../kotlin/com/powersync/AttachmentsTest.kt | 82 +++++++++++++++++++ .../testutils/TestAttachmentsQueue.kt | 3 + .../com/powersync/attachments/README.md | 1 - 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 6d230c20..67fb86bf 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -5,12 +5,16 @@ import co.touchlab.kermit.ExperimentalKermitApi import com.powersync.attachments.Attachment import com.powersync.attachments.AttachmentState import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.SyncErrorHandler import com.powersync.attachments.createAttachmentsTable import com.powersync.db.schema.Schema import com.powersync.testutils.MockedRemoteStorage import com.powersync.testutils.TestAttachmentsQueue import com.powersync.testutils.UserRow +import dev.mokkery.answering.throws +import dev.mokkery.everySuspend import dev.mokkery.matcher.any +import dev.mokkery.mock import dev.mokkery.spy import dev.mokkery.verifySuspend import kotlinx.coroutines.flow.flowOf @@ -367,6 +371,84 @@ class AttachmentsTest { actual = attachmentRecord.state, ) + attachmentQuery.cancel() + queue.close() + } + } + + @Test + fun testSkipFailedDownload() = + runTest { + turbineScope { + val remote = + mock { + everySuspend { downloadFile(any()) } throws (Exception("Test error")) + } + + val queue = + TestAttachmentsQueue( + db = database, + remoteStorage = remote, + archivedCacheLimit = 0, + errorHandler = + object : SyncErrorHandler { + override suspend fun onDownloadError( + attachment: Attachment, + exception: Exception, + ): Boolean = false + + override suspend fun onUploadError( + attachment: Attachment, + exception: Exception, + ): Boolean = false + + override suspend fun onDeleteError( + attachment: Attachment, + exception: Exception, + ): Boolean = false + }, + ) + + queue.start() + + // Monitor the attachments table for testing + val attachmentQuery = + database + .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } + .testIn(this) + + val result = attachmentQuery.awaitItem() + + // There should not be any attachment records here + assertEquals(expected = 0, actual = result.size) + + // Create a user with a photo_id specified. + // This code did not save an attachment before assigning a photo_id. + // This is equivalent to requiring an attachment download + database.execute( + """ + INSERT INTO + users (id, name, email, photo_id) + VALUES + (uuid(), "steven", "steven@journeyapps.com", uuid()) + """, + ) + + var attachmentRecord = attachmentQuery.awaitItem().first() + assertNotNull(attachmentRecord) + + assertEquals( + expected = AttachmentState.QUEUED_DOWNLOAD.ordinal, + actual = attachmentRecord.state, + ) + + // The download should fail. We don't specify a retry. The record should be archived. + attachmentRecord = attachmentQuery.awaitItem().first() + assertEquals( + expected = AttachmentState.ARCHIVED.ordinal, + actual = attachmentRecord.state, + ) + attachmentQuery.cancel() queue.close() } diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt index 4b407271..143aad65 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt @@ -3,6 +3,7 @@ package com.powersync.testutils import com.powersync.PowerSyncDatabase import com.powersync.attachments.AbstractAttachmentQueue import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.SyncErrorHandler import com.powersync.attachments.WatchedAttachmentItem import com.powersync.db.getString import kotlinx.coroutines.flow.Flow @@ -11,10 +12,12 @@ internal class TestAttachmentsQueue( db: PowerSyncDatabase, remoteStorage: RemoteStorageAdapter, archivedCacheLimit: Long, + errorHandler: SyncErrorHandler? = null, ) : AbstractAttachmentQueue( db, remoteStorage, archivedCacheLimit = archivedCacheLimit, + errorHandler = errorHandler, ) { override fun watchAttachments(): Flow> = db.watch( diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index cdc6085b..b173186f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -168,7 +168,6 @@ queue.saveFile( * This lambda is invoked in the same transaction which creates the attachment record. * Assignments of the newly created photo_id should be done in the same transaction for maximum efficiency. */ - // Set the photo_id of a checklist to the attachment id tx.execute( """ UPDATE From 408db0b9d77c551b5c215ee516b5647cdbcbcd86 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 3 Apr 2025 16:35:36 +0200 Subject: [PATCH 09/51] fix tests --- .../kotlin/com/powersync/AttachmentsTest.kt | 11 ++++++++--- .../powersync/attachments/AbstractAttachmentQueue.kt | 5 ++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 67fb86bf..10b50f73 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -13,7 +13,9 @@ import com.powersync.testutils.TestAttachmentsQueue import com.powersync.testutils.UserRow import dev.mokkery.answering.throws import dev.mokkery.everySuspend +import dev.mokkery.matcher.ArgMatchersScope import dev.mokkery.matcher.any +import dev.mokkery.matcher.matching import dev.mokkery.mock import dev.mokkery.spy import dev.mokkery.verifySuspend @@ -127,7 +129,7 @@ class AttachmentsTest { assertEquals(expected = AttachmentState.SYNCED.ordinal, attachmentRecord.state) // A download should have been attempted for this file - verifySuspend { remote.downloadFile(attachmentRecord) } + verifySuspend { remote.downloadFile(attachmentMatcher(attachmentRecord)) } // A file should now exist val localUri = attachmentRecord.localUri!! @@ -233,7 +235,7 @@ class AttachmentsTest { verifySuspend { remote.uploadFile( any(), - attachmentRecord, + attachmentMatcher(attachmentRecord), ) } @@ -332,7 +334,7 @@ class AttachmentsTest { assertEquals(expected = AttachmentState.SYNCED.ordinal, attachmentRecord.state) // A download should have been attempted for this file - verifySuspend { remote.downloadFile(attachmentRecord) } + verifySuspend { remote.downloadFile(attachmentMatcher(attachmentRecord)) } // A file should now exist val localUri = attachmentRecord.localUri!! @@ -454,3 +456,6 @@ class AttachmentsTest { } } } + +fun ArgMatchersScope.attachmentMatcher(attachment: Attachment): Attachment = + matching(toString = { "attachment($attachment)" }, predicate = { it.id == attachment.id }) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt index 414df41b..3b6a0177 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt @@ -160,7 +160,6 @@ public abstract class AbstractAttachmentQueue( localStorage.makeDir(Path(getStorageDirectory(), subdirectory).toString()) } - // Start watching for changes val scope = CoroutineScope(Dispatchers.IO) syncingService.startPeriodicSync(syncInterval) @@ -169,7 +168,7 @@ public abstract class AbstractAttachmentQueue( syncStatusJob = scope.launch { val statusJob = - scope.launch { + launch { var previousConnected = db.currentStatus.connected db.currentStatus.asFlow().collect { status -> if (!previousConnected && status.connected) { @@ -180,7 +179,7 @@ public abstract class AbstractAttachmentQueue( } val watchJob = - scope.launch { + launch { // Watch local attachment relationships and sync the attachment records watchAttachments().collect { items -> processWatchedAttachments(items) From 6fafc3e10116808e6f4aba32ee7aa9f085f7463b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 3 Apr 2025 17:03:42 +0200 Subject: [PATCH 10/51] cleanup tests --- .../kotlin/com/powersync/DatabaseTest.kt | 9 ++++++++- .../com/powersync/testutils/TestAttachmentsQueue.kt | 2 +- .../kotlin/com/powersync/testutils/TestUtils.kt | 4 +++- .../kotlin/com/powersync/sync/SyncStream.kt | 5 ----- .../testutils/TestUtils.iosSimulatorArm64.kt | 11 ++++------- .../kotlin/com/powersync/testutils/TestUtils.jvm.kt | 4 +++- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt index 7e4614ca..2273d075 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt @@ -12,6 +12,7 @@ import com.powersync.db.schema.Schema import com.powersync.testutils.UserRow import com.powersync.testutils.generatePrintLogWriter import com.powersync.testutils.getTempDir +import com.powersync.testutils.isIOS import com.powersync.testutils.waitFor import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.Dispatchers @@ -294,7 +295,13 @@ class DatabaseTest { factory = com.powersync.testutils.factory, schema = Schema(UserRow.table), dbFilename = dbFilename, - dbDirectory = getTempDir(), + dbDirectory = + if (isIOS()) { + // Not supported for iOS + null + } else { + getTempDir() + }, logger = logger, ) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt index 143aad65..40473759 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt @@ -37,5 +37,5 @@ internal class TestAttachmentsQueue( /** * For tests this uses a temporary directory. On iOS it uses the user storage directory */ - override fun getStorageDirectory(): String = getTempDir() ?: super.getStorageDirectory() + override fun getStorageDirectory(): String = getTempDir() } diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestUtils.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestUtils.kt index e10cb099..47f2f62d 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestUtils.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestUtils.kt @@ -8,7 +8,9 @@ expect val factory: DatabaseDriverFactory expect fun cleanup(path: String) -expect fun getTempDir(): String? +expect fun getTempDir(): String + +expect fun isIOS(): Boolean fun generatePrintLogWriter() = object : LogWriter() { diff --git a/core/src/commonMain/kotlin/com/powersync/sync/SyncStream.kt b/core/src/commonMain/kotlin/com/powersync/sync/SyncStream.kt index 23b3e5ed..e4c999df 100644 --- a/core/src/commonMain/kotlin/com/powersync/sync/SyncStream.kt +++ b/core/src/commonMain/kotlin/com/powersync/sync/SyncStream.kt @@ -91,11 +91,6 @@ internal class SyncStream( invalidCredentials = false } streamingSyncIteration() -// val state = streamingSyncIteration() -// TODO: We currently always retry -// if (!state.retry) { -// break; -// } } catch (e: Exception) { if (e is CancellationException) { throw e diff --git a/core/src/iosSimulatorArm64Test/kotlin/com/powersync/testutils/TestUtils.iosSimulatorArm64.kt b/core/src/iosSimulatorArm64Test/kotlin/com/powersync/testutils/TestUtils.iosSimulatorArm64.kt index 1a4e93ea..396be545 100644 --- a/core/src/iosSimulatorArm64Test/kotlin/com/powersync/testutils/TestUtils.iosSimulatorArm64.kt +++ b/core/src/iosSimulatorArm64Test/kotlin/com/powersync/testutils/TestUtils.iosSimulatorArm64.kt @@ -3,6 +3,7 @@ package com.powersync.testutils import com.powersync.DatabaseDriverFactory import kotlinx.io.files.Path import kotlinx.io.files.SystemFileSystem +import platform.Foundation.NSTemporaryDirectory actual val factory: DatabaseDriverFactory get() = DatabaseDriverFactory() @@ -14,10 +15,6 @@ actual fun cleanup(path: String) { } } -/** - * We could use SystemTemporaryDirectory here in future, but we return null here - * to skip tests which rely on a temporary directory for iOS. - * The reason for skipping these tests is that the SQLiteR library does not currently - * support opening DB paths for custom directories. - */ -actual fun getTempDir(): String? = null +actual fun getTempDir(): String = NSTemporaryDirectory() + +actual fun isIOS(): Boolean = true diff --git a/core/src/jvmTest/kotlin/com/powersync/testutils/TestUtils.jvm.kt b/core/src/jvmTest/kotlin/com/powersync/testutils/TestUtils.jvm.kt index 3b53926a..9296387a 100644 --- a/core/src/jvmTest/kotlin/com/powersync/testutils/TestUtils.jvm.kt +++ b/core/src/jvmTest/kotlin/com/powersync/testutils/TestUtils.jvm.kt @@ -10,4 +10,6 @@ actual fun cleanup(path: String) { File(path).delete() } -actual fun getTempDir(): String? = System.getProperty("java.io.tmpdir") +actual fun getTempDir(): String = System.getProperty("java.io.tmpdir") + +actual fun isIOS(): Boolean = false From 6530ed06133abc80c68509eef463dec4b3ac344d Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 3 Apr 2025 17:16:43 +0200 Subject: [PATCH 11/51] cleanup deletes --- .../attachments/AbstractAttachmentQueue.kt | 19 ++++++++++++------- .../com/powersync/attachments/README.md | 7 +++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt index 3b6a0177..e3f9924b 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt @@ -322,10 +322,14 @@ public abstract class AbstractAttachmentQueue( } } - // Archive any items not specified in the watched items + /** + * Archive any items not specified in the watched items except for items pending delete. + */ currentAttachments - .filter { null == items.find { update -> update.id == it.id } } - .forEach { + .filter { + it.state != AttachmentState.QUEUED_DELETE.ordinal && + null == items.find { update -> update.id == it.id } + }.forEach { attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED.ordinal)) } @@ -345,12 +349,12 @@ public abstract class AbstractAttachmentQueue( * This method can be overriden for custom behaviour. */ @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun saveFile( + public open suspend fun saveFile( data: Flow, mediaType: String, fileExtension: String?, - updateHook: (context: ConnectionContext, attachment: Attachment) -> R, - ): R = + updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, + ): Attachment = runWrappedSuspending { val id = db.get("SELECT uuid()") { it.getString(0)!! } val filename = @@ -380,12 +384,13 @@ public abstract class AbstractAttachmentQueue( // Allow consumers to set relationships to this attachment id updateHook(tx, attachment) + return@writeTransaction attachment } } /** * A function which creates an attachment delete operation locally. This operation is queued - * for delete after creating. + * for delete. * The default implementation assumes the attachment record already exists locally. An exception * is thrown if the record does not exist locally. * This method can be overriden for custom behaviour. diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index b173186f..8527cf6f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -223,8 +223,6 @@ By default, this is every 30 seconds, but can be configured by setting `syncInte ### Downloading - An `AttachmentRecord` is created or updated with `QUEUED_DOWNLOAD` state. -- The watched query from `watchAttachments` adds the attachment `id` into a queue of IDs to download - and triggers the download process - The watched query from `watchAttachments` adds the attachment `id` into a queue of IDs to download and triggers the download process - If the photo is not on the device, it is downloaded from cloud storage. @@ -236,9 +234,10 @@ By default, this is every 30 seconds, but can be configured by setting `syncInte When an attachment is deleted by a user action or cache expiration: -- Related `AttachmentRecord` is removed from attachments table. +- An `AttachmentRecord` is created or updated with `QUEUED_DELETE` state. +- The `RemoteStorage`'s `deleteFile` method will be called. - Local file (if exists) is deleted. -- File on cloud storage is deleted. +- If successful, the `AttachmentRecord` is deleted. ### Expire Cache From 561c02ad6273dcaff2853af461e29ba0136ae04d Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 4 Apr 2025 17:40:25 +0200 Subject: [PATCH 12/51] wip: Android Attachments Demo --- connectors/supabase/build.gradle.kts | 1 + .../connector/supabase/SupabaseConnector.kt | 25 +++- .../AbstractLocalStorageAdapter.android.kt | 5 +- .../kotlin/com/powersync/AttachmentsTest.kt | 8 +- .../attachments/AbstractAttachmentQueue.kt | 57 +++++---- .../com/powersync/attachments/README.md | 4 +- .../storage/IOLocalStorageAdapter.kt | 24 ++-- .../attachments/sync/SyncingService.kt | 4 + .../app/build.gradle.kts | 23 +++- .../app/src/main/AndroidManifest.xml | 13 ++- .../java/com/powersync/androidexample/App.kt | 54 +++++---- .../androidexample/AttachmentsQueue.kt | 23 ++++ .../java/com/powersync/androidexample/Auth.kt | 7 +- .../powersync/androidexample/MainActivity.kt | 16 ++- .../androidexample/SupabaseRemoteStorage.kt | 33 ++++++ .../androidexample/components/EditDialog.kt | 39 +++++++ .../androidexample/powersync/Schema.kt | 3 + .../androidexample/powersync/Todo.kt | 110 ++++++++++++++---- .../androidexample/screens/TodosScreen.kt | 2 +- .../androidexample/ui/CameraService.kt | 52 +++++++++ .../app/src/main/res/xml/filepaths.xml | 4 + .../main/res/xml/network_security_config.xml | 6 + .../gradle/libs.versions.toml | 2 +- .../local.properties.example | 1 + gradle/libs.versions.toml | 2 +- 25 files changed, 416 insertions(+), 102 deletions(-) create mode 100644 demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/AttachmentsQueue.kt create mode 100644 demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt create mode 100644 demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/ui/CameraService.kt create mode 100644 demos/android-supabase-todolist/app/src/main/res/xml/filepaths.xml create mode 100644 demos/android-supabase-todolist/app/src/main/res/xml/network_security_config.xml diff --git a/connectors/supabase/build.gradle.kts b/connectors/supabase/build.gradle.kts index fb1f282e..9309dfb4 100644 --- a/connectors/supabase/build.gradle.kts +++ b/connectors/supabase/build.gradle.kts @@ -36,6 +36,7 @@ kotlin { implementation(libs.kotlinx.coroutines.core) implementation(libs.supabase.client) api(libs.supabase.auth) + api(libs.supabase.storage) } } } diff --git a/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseConnector.kt b/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseConnector.kt index f4c33adb..388b70e6 100644 --- a/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseConnector.kt +++ b/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseConnector.kt @@ -17,6 +17,9 @@ import io.github.jan.supabase.auth.user.UserSession import io.github.jan.supabase.createSupabaseClient import io.github.jan.supabase.postgrest.Postgrest import io.github.jan.supabase.postgrest.from +import io.github.jan.supabase.storage.BucketApi +import io.github.jan.supabase.storage.Storage +import io.github.jan.supabase.storage.storage import io.ktor.client.plugins.HttpSend import io.ktor.client.plugins.plugin import io.ktor.client.statement.bodyAsText @@ -31,6 +34,7 @@ import kotlinx.serialization.json.Json public class SupabaseConnector( public val supabaseClient: SupabaseClient, public val powerSyncEndpoint: String, + private val storageBucket: String? = null, ) : PowerSyncBackendConnector() { private var errorCode: String? = null @@ -52,17 +56,29 @@ public class SupabaseConnector( } } + public fun storageBucket(): BucketApi { + if (storageBucket == null) { + throw Exception("No bucket has been specified") + } + return supabaseClient.storage[storageBucket] + } + public constructor( supabaseUrl: String, supabaseKey: String, powerSyncEndpoint: String, + storageBucket: String? = null, ) : this( supabaseClient = createSupabaseClient(supabaseUrl, supabaseKey) { install(Auth) install(Postgrest) + if (storageBucket != null) { + install(Storage) + } }, powerSyncEndpoint = powerSyncEndpoint, + storageBucket = storageBucket, ) init { @@ -81,7 +97,10 @@ public class SupabaseConnector( val responseText = response.bodyAsText() try { - val error = Json { coerceInputValues = true }.decodeFromString>(responseText) + val error = + Json { coerceInputValues = true }.decodeFromString>( + responseText, + ) errorCode = error["code"] } catch (e: Exception) { Logger.e("Failed to parse error response: $e") @@ -139,7 +158,9 @@ public class SupabaseConnector( check(supabaseClient.auth.sessionStatus.value is SessionStatus.Authenticated) { "Supabase client is not authenticated" } // Use Supabase token for PowerSync - val session = supabaseClient.auth.currentSessionOrNull() ?: error("Could not fetch Supabase credentials") + val session = + supabaseClient.auth.currentSessionOrNull() + ?: error("Could not fetch Supabase credentials") check(session.user != null) { "No user data" } diff --git a/core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt b/core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt index 5e78847c..f890caec 100644 --- a/core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt +++ b/core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt @@ -1,9 +1,12 @@ package com.powersync.attachments.storage import android.os.Environment +import android.os.Environment.DIRECTORY_DOCUMENTS import com.powersync.attachments.LocalStorageAdapter +import okio.FileSystem + @Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { - actual override fun getUserStorageDirectory(): String = Environment.getDataDirectory().absolutePath + actual override fun getUserStorageDirectory(): String = FileSystem.SYSTEM_TEMPORARY_DIRECTORY.normalized().toString() } diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 10b50f73..4136fb39 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -80,7 +80,7 @@ class AttachmentsTest { archivedCacheLimit = 0, ) - queue.start() + queue.startSync() // Monitor the attachments table for testing val attachmentQuery = @@ -184,7 +184,7 @@ class AttachmentsTest { archivedCacheLimit = 0, ) - queue.start() + queue.startSync() // Monitor the attachments table for testing val attachmentQuery = @@ -285,7 +285,7 @@ class AttachmentsTest { archivedCacheLimit = 10, ) - queue.start() + queue.startSync() // Monitor the attachments table for testing val attachmentQuery = @@ -411,7 +411,7 @@ class AttachmentsTest { }, ) - queue.start() + queue.startSync() // Monitor the attachments table for testing val attachmentQuery = diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt index e3f9924b..8474443c 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt @@ -88,6 +88,10 @@ public abstract class AbstractAttachmentQueue( * Records are deleted once the number of items exceeds this value. */ private val archivedCacheLimit: Long = 100, + /** + * Throttles remote sync operations triggering + */ + private val syncThrottleDuration: Duration = 1.seconds, /** * Creates a list of subdirectories in the {attachmentDirectoryName} directory */ @@ -133,6 +137,7 @@ public abstract class AbstractAttachmentQueue( ::getLocalUri, errorHandler, logger, + syncThrottleDuration, ) private var syncStatusJob: Job? = null @@ -147,7 +152,7 @@ public abstract class AbstractAttachmentQueue( * 3. Adding trigger to run uploads, downloads, and deletes when device is online after being offline */ @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun start(): Unit = + public suspend fun startSync(): Unit = runWrappedSuspending { mutex.withLock { if (closed) { @@ -353,7 +358,7 @@ public abstract class AbstractAttachmentQueue( data: Flow, mediaType: String, fileExtension: String?, - updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, + updateHook: ((context: ConnectionContext, attachment: Attachment) -> Unit)? = null, ): Attachment = runWrappedSuspending { val id = db.get("SELECT uuid()") { it.getString(0)!! } @@ -369,22 +374,24 @@ public abstract class AbstractAttachmentQueue( * assignment should happen in the same transaction. */ db.writeTransaction { tx -> - val attachment = - attachmentsService.upsertAttachment( - Attachment( - id = id, - filename = filename, - size = fileSize, - mediaType = mediaType, - state = AttachmentState.QUEUED_UPLOAD.ordinal, - localUri = localUri, - ), - tx, - ) - - // Allow consumers to set relationships to this attachment id - updateHook(tx, attachment) - return@writeTransaction attachment + val attachment = Attachment( + id = id, + filename = filename, + size = fileSize, + mediaType = mediaType, + state = AttachmentState.QUEUED_UPLOAD.ordinal, + localUri = localUri, + ) + + /** + * Allow consumers to set relationships to this attachment id + */ + updateHook?.invoke(tx, attachment) + + return@writeTransaction attachmentsService.upsertAttachment( + attachment, + tx, + ) } } @@ -396,15 +403,21 @@ public abstract class AbstractAttachmentQueue( * This method can be overriden for custom behaviour. */ @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun deleteFile(attachmentId: String): Attachment = + public open suspend fun deleteFile(attachmentId: String, + updateHook: ((context: ConnectionContext, attachment: Attachment) -> Unit)? = null, + ): Attachment = runWrappedSuspending { val attachment = attachmentsService.getAttachment(attachmentId) ?: throw Exception("Attachment record with id $attachmentId was not found.") - return@runWrappedSuspending attachmentsService.saveAttachment( - attachment.copy(state = AttachmentState.QUEUED_DELETE.ordinal), - ) + db.writeTransaction { tx -> + updateHook?.invoke(tx, attachment) + return@writeTransaction attachmentsService.upsertAttachment( + attachment.copy(state = AttachmentState.QUEUED_DELETE.ordinal), + tx, + ) + } } /** diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index 8527cf6f..b14cceee 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -141,7 +141,7 @@ val remote = object : RemoteStorageAdapter() { ``` -5. Instantiate a new `AttachmentQueue` and call `start()` to start syncing attachments. +5. Instantiate a new `AttachmentQueue` and call `startSync()` to start syncing attachments. ```kotlin val queue = @@ -150,7 +150,7 @@ val remote = object : RemoteStorageAdapter() { remoteStorage = remote, ) -queue.start() +queue.startSync() ``` 7. Finally, to create an attachment and add it to the queue, call `saveFile()`. This method will diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt index c8e121ce..0db5109f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt @@ -2,11 +2,13 @@ package com.powersync.attachments.storage import com.powersync.attachments.storage.AbstractLocalStorageAdapter import com.powersync.db.runWrappedSuspending +import io.ktor.utils.io.core.readBytes import io.ktor.utils.io.core.remaining import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.withContext import kotlinx.io.Buffer import kotlinx.io.buffered @@ -48,23 +50,19 @@ public class IOLocalStorageAdapter : AbstractLocalStorageAdapter() { mediaType: String?, ): Flow = flow { - withContext(Dispatchers.IO) { SystemFileSystem.source(Path(filePath)).use { source -> source.buffered().use { bufferedSource -> - val bufferSize = 8192L // Read in 8KB chunks - while (bufferedSource.remaining > 0) { - val byteCount = - min( - bufferedSource.remaining, - bufferSize, - ) - val bytesRead = bufferedSource.readByteArray(byteCount.toInt()) - emit(bytesRead) - } + var remaining = 0L + val bufferSize = 8192L + do { + bufferedSource.request(bufferSize) + remaining = bufferedSource.remaining + emit(bufferedSource.readBytes(remaining.toInt())) + } while (remaining > 0) + } } - } - } + }.flowOn(Dispatchers.IO) public override suspend fun deleteFile(filePath: String): Unit = runWrappedSuspending { diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index c79f8505..183a8c61 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -8,6 +8,7 @@ import com.powersync.attachments.AttachmentState import com.powersync.attachments.LocalStorageAdapter import com.powersync.attachments.RemoteStorageAdapter import com.powersync.attachments.SyncErrorHandler +import com.powersync.utils.throttle import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -23,6 +24,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds /** * Service used to sync attachments between local and remote storage @@ -34,6 +36,7 @@ internal class SyncingService( private val getLocalUri: suspend (String) -> String, private val errorHandler: SyncErrorHandler?, private val logger: Logger, + private val syncThrottle: Duration = 5.seconds, ) { private val scope = CoroutineScope(Dispatchers.IO) private val mutex = Mutex() @@ -60,6 +63,7 @@ internal class SyncingService( // while we are processing. We will always process on the trailing edge. // This buffer operation should automatically be applied to all merged sources. .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) + .throttle(syncThrottle.inWholeMilliseconds) .collect { /** * Gets and performs the operations for active attachments which are diff --git a/demos/android-supabase-todolist/app/build.gradle.kts b/demos/android-supabase-todolist/app/build.gradle.kts index 7b24a362..24c390b4 100644 --- a/demos/android-supabase-todolist/app/build.gradle.kts +++ b/demos/android-supabase-todolist/app/build.gradle.kts @@ -13,9 +13,10 @@ if (localPropertiesFile.exists()) { localPropertiesFile.inputStream().use { localProperties.load(it) } } -fun getLocalProperty(key: String, defaultValue: String): String { - return localProperties.getProperty(key, defaultValue) -} +fun getLocalProperty( + key: String, + defaultValue: String, +): String = localProperties.getProperty(key, defaultValue) android { namespace = "com.powersync.androidexample" @@ -38,7 +39,16 @@ android { } buildConfigField("String", "SUPABASE_URL", "\"${getLocalProperty("SUPABASE_URL", "")}\"") - buildConfigField("String", "SUPABASE_ANON_KEY", "\"${getLocalProperty("SUPABASE_ANON_KEY", "")}\"") + buildConfigField( + "String", + "SUPABASE_ANON_KEY", + "\"${getLocalProperty("SUPABASE_ANON_KEY", "")}\"", + ) + buildConfigField( + "String", + "SUPABASE_ATTACHMENT_BUCKET", + "\"${getLocalProperty("SUPABASE_ATTACHMENT_BUCKET", "")}\"", + ) buildConfigField("String", "POWERSYNC_URL", "\"${getLocalProperty("POWERSYNC_URL", "")}\"") } @@ -47,7 +57,7 @@ android { isMinifyEnabled = false proguardFiles( getDefaultProguardFile("proguard-android-optimize.txt"), - "proguard-rules.pro" + "proguard-rules.pro", ) } } @@ -96,4 +106,5 @@ dependencies { implementation(libs.uuid) implementation(libs.kermit) implementation(libs.androidx.material.icons.extended) -} \ No newline at end of file + implementation("androidx.activity:activity-ktx:1.9.0") +} diff --git a/demos/android-supabase-todolist/app/src/main/AndroidManifest.xml b/demos/android-supabase-todolist/app/src/main/AndroidManifest.xml index 622607af..844bfbac 100644 --- a/demos/android-supabase-todolist/app/src/main/AndroidManifest.xml +++ b/demos/android-supabase-todolist/app/src/main/AndroidManifest.xml @@ -11,7 +11,9 @@ android:roundIcon="@mipmap/ic_launcher_round" android:supportsRtl="true" android:theme="@style/Theme.App.Starting" - tools:targetApi="31"> + tools:targetApi="31" + android:networkSecurityConfig="@xml/network_security_config" + > + + + \ No newline at end of file diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index 637062c6..69c94f28 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -11,8 +11,11 @@ import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberUpdatedState import androidx.compose.ui.Modifier -import com.powersync.androidexample.BuildConfig import com.powersync.PowerSyncDatabase +import com.powersync.androidexample.AttachmentsQueue +import com.powersync.androidexample.BuildConfig +import com.powersync.androidexample.SupabaseRemoteStorage +import com.powersync.androidexample.ui.CameraService import com.powersync.compose.rememberDatabaseDriverFactory import com.powersync.connector.supabase.SupabaseConnector import com.powersync.demos.components.EditDialog @@ -26,25 +29,31 @@ import com.powersync.demos.screens.SignUpScreen import com.powersync.demos.screens.TodosScreen import kotlinx.coroutines.runBlocking - @Composable -fun App() { +fun App(cameraService: CameraService) { val driverFactory = rememberDatabaseDriverFactory() - val supabase = remember { - SupabaseConnector( - powerSyncEndpoint = BuildConfig.POWERSYNC_URL, - supabaseUrl = BuildConfig.SUPABASE_URL, - supabaseKey = BuildConfig.SUPABASE_ANON_KEY - ) - } - val db = remember { PowerSyncDatabase(driverFactory, schema) } + val supabase = + remember { + SupabaseConnector( + powerSyncEndpoint = BuildConfig.POWERSYNC_URL, + supabaseUrl = BuildConfig.SUPABASE_URL, + supabaseKey = BuildConfig.SUPABASE_ANON_KEY, + storageBucket = BuildConfig.SUPABASE_ATTACHMENT_BUCKET, + ) + } + + val db = remember { PowerSyncDatabase(driverFactory, schema, dbFilename = "222.sqlite") } + val attachments = + remember { AttachmentsQueue(db = db, remoteStorage = SupabaseRemoteStorage(supabase)) } + val syncStatus = db.currentStatus val status by syncStatus.asFlow().collectAsState(syncStatus) val navController = remember { NavController(Screen.Home) } - val authViewModel = remember { - AuthViewModel(supabase, db, navController) - } + val authViewModel = + remember { + AuthViewModel(supabase, db, attachments, navController) + } val authState by authViewModel.authState.collectAsState() val currentScreen by navController.currentScreen.collectAsState() @@ -59,9 +68,9 @@ fun App() { val items by lists.value.watchItems().collectAsState(initial = emptyList()) val listsInputText by lists.value.inputText.collectAsState() - val todos = remember { mutableStateOf(Todo(db, userId)) } + val todos = remember { mutableStateOf(Todo(db, attachments, userId)) } LaunchedEffect(currentUserId.value) { - todos.value = Todo(db, currentUserId.value) + todos.value = Todo(db, attachments, currentUserId.value) } val todoItems by todos.value.watchItems(selectedListId).collectAsState(initial = emptyList()) val editingItem by todos.value.editingItem.collectAsState() @@ -75,7 +84,7 @@ fun App() { when (currentScreen) { is Screen.Home -> { - if(authState == AuthState.SignedOut) { + if (authState == AuthState.SignedOut) { navController.navigate(Screen.SignIn) } @@ -121,29 +130,32 @@ fun App() { onCloseClicked = todos.value::onEditorCloseClicked, onTextChanged = todos.value::onEditorTextChanged, onDoneChanged = todos.value::onEditorDoneChanged, + onPhotoClear = todos.value::onPhotoDelete, + onPhotoCapture = {todos.value::onPhotoCapture.invoke(cameraService)} + ) } } is Screen.SignIn -> { - if(authState == AuthState.SignedIn) { + if (authState == AuthState.SignedIn) { navController.navigate(Screen.Home) } SignInScreen( navController, - authViewModel + authViewModel, ) } is Screen.SignUp -> { - if(authState == AuthState.SignedIn) { + if (authState == AuthState.SignedIn) { navController.navigate(Screen.Home) } SignUpScreen( navController, - authViewModel + authViewModel, ) } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/AttachmentsQueue.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/AttachmentsQueue.kt new file mode 100644 index 00000000..b9827392 --- /dev/null +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/AttachmentsQueue.kt @@ -0,0 +1,23 @@ +package com.powersync.androidexample + +import com.powersync.PowerSyncDatabase +import com.powersync.attachments.AbstractAttachmentQueue +import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.WatchedAttachmentItem +import com.powersync.db.getString +import kotlinx.coroutines.flow.Flow + +class AttachmentsQueue(db: PowerSyncDatabase, + remoteStorage: RemoteStorageAdapter +) : AbstractAttachmentQueue(db, remoteStorage, archivedCacheLimit = 0) { + override fun watchAttachments(): Flow> { + return db.watch( + "SELECT photo_id from todos WHERE photo_id IS NOT NULL" + ) { + WatchedAttachmentItem( + id = it.getString("photo_id"), + fileExtension = "jpg" + ) + } + } +} \ No newline at end of file diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt index 889b3456..87633a74 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt @@ -4,6 +4,7 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase +import com.powersync.androidexample.AttachmentsQueue import com.powersync.connector.supabase.SupabaseConnector import io.github.jan.supabase.auth.status.RefreshFailureCause import io.github.jan.supabase.auth.status.SessionStatus @@ -20,6 +21,7 @@ sealed class AuthState { internal class AuthViewModel( private val supabase: SupabaseConnector, private val db: PowerSyncDatabase, + private val attachmentsQueue: AttachmentsQueue, private val navController: NavController, ) : ViewModel() { private val _authState = MutableStateFlow(AuthState.SignedOut) @@ -35,9 +37,10 @@ internal class AuthViewModel( _authState.value = AuthState.SignedIn _userId.value = it.session.user?.id db.connect(supabase) + attachmentsQueue.startSync() navController.navigate(Screen.Home) } - SessionStatus.Initializing -> Logger.e("Loading from storage") + is SessionStatus.Initializing -> Logger.e("Loading from storage") is SessionStatus.RefreshFailure -> { when (it.cause) { is RefreshFailureCause.NetworkError -> Logger.e("Network error occurred") @@ -72,6 +75,8 @@ internal class AuthViewModel( suspend fun signOut() { try { + attachmentsQueue.clearQueue() + attachmentsQueue.close() supabase.signOut() } catch (e: Exception) { Logger.e("Error signing out: $e") diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt index 39e1eceb..d253a3ac 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt @@ -1,19 +1,31 @@ package com.powersync.androidexample +import android.net.Uri import android.os.Bundle +import android.os.Environment import androidx.activity.ComponentActivity import androidx.activity.compose.setContent +import androidx.activity.result.contract.ActivityResultContracts +import androidx.core.content.FileProvider import androidx.core.splashscreen.SplashScreen.Companion.installSplashScreen +import com.powersync.androidexample.ui.CameraService import com.powersync.demos.App +import kotlinx.coroutines.CompletableDeferred +import java.io.File +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale class MainActivity : ComponentActivity() { + private val cameraService = CameraService(this) override fun onCreate(savedInstanceState: Bundle?) { installSplashScreen() super.onCreate(savedInstanceState) setContent { - App() + App(cameraService = cameraService) } } -} \ No newline at end of file + +} diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt new file mode 100644 index 00000000..743a9676 --- /dev/null +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt @@ -0,0 +1,33 @@ +package com.powersync.androidexample + +import com.powersync.attachments.Attachment +import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.connector.supabase.SupabaseConnector +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOf + +class SupabaseRemoteStorage( + val connector: SupabaseConnector, +) : RemoteStorageAdapter { + override suspend fun uploadFile( + fileData: Flow, + attachment: Attachment, + ) { + // Supabase wants a single ByteArray + val buffer = ByteArray(attachment.size!!.toInt()) + var position = 0 + fileData.collect { + System.arraycopy(it, 0, buffer, position, it.size) + position += it.size + } + + connector.storageBucket().upload(attachment.filename, buffer) + } + + override suspend fun downloadFile(attachment: Attachment): Flow = flowOf(connector.storageBucket().downloadAuthenticated(attachment.filename)) + + override suspend fun deleteFile(attachment: Attachment) { + connector.storageBucket().delete(attachment.filename) + } +} diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt index 6df9327d..c4b56ba0 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt @@ -1,8 +1,12 @@ package com.powersync.demos.components +import android.graphics.BitmapFactory +import androidx.compose.foundation.Image +import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.IntrinsicSize +import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxWidth @@ -18,8 +22,11 @@ import androidx.compose.material3.ProvideTextStyle import androidx.compose.material3.Text import androidx.compose.material3.TextField import androidx.compose.runtime.Composable +import androidx.compose.runtime.remember import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.unit.dp import androidx.compose.ui.window.Dialog import com.powersync.demos.powersync.TodoItem @@ -30,6 +37,8 @@ internal fun EditDialog( onCloseClicked: () -> Unit, onTextChanged: (String) -> Unit, onDoneChanged: (Boolean) -> Unit, + onPhotoClear: () -> Unit, + onPhotoCapture: () -> Unit ) { EditDialog( onCloseRequest = onCloseClicked, @@ -51,6 +60,36 @@ internal fun EditDialog( onCheckedChange = onDoneChanged, ) } + + val bitmap = remember(item.photoURI) { + item.photoURI?.let { BitmapFactory.decodeFile(it)?.asImageBitmap() } + } + + Box( + modifier = Modifier + .clickable { if (item.photoId == null) onPhotoCapture() } + .padding(8.dp), + contentAlignment = Alignment.Center + ) { + if (bitmap == null) { + Button( + onClick = onPhotoCapture, + modifier = Modifier.align(Alignment.Center), + contentPadding = PaddingValues(0.dp) + ) { + Text("Add Photo", color = Color.Gray) + } + } else { + Image(bitmap = bitmap, contentDescription = "Photo Preview") + Button( + onClick = onPhotoClear, + modifier = Modifier.align(Alignment.TopEnd), + contentPadding = PaddingValues(0.dp) + ) { + Text("Clear Photo", color = Color.Red) + } + } + } } } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Schema.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Schema.kt index 9ab757e4..8ce17635 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Schema.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Schema.kt @@ -1,5 +1,6 @@ package com.powersync.demos.powersync +import com.powersync.attachments.createAttachmentsTable import com.powersync.db.schema.Column import com.powersync.db.schema.Index import com.powersync.db.schema.IndexedColumn @@ -43,6 +44,7 @@ val schema: Schema = Schema( listOf( todos, lists, + createAttachmentsTable("attachments") ) ) @@ -57,6 +59,7 @@ data class TodoItem( val id: String, val listId: String, val photoId: String?, + val photoURI: String?, val createdAt: String?, val completedAt: String?, val description: String, diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt index 2bfa54fe..c6edae35 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt @@ -1,37 +1,47 @@ package com.powersync.demos.powersync +import androidx.activity.result.contract.ActivityResultContracts import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase +import com.powersync.androidexample.AttachmentsQueue +import com.powersync.androidexample.ui.CameraService import com.powersync.db.getLongOptional import com.powersync.db.getString import com.powersync.db.getStringOptional +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.launch import kotlinx.datetime.Clock internal class Todo( private val db: PowerSyncDatabase, - private val userId: String? -): ViewModel() { - + private val attachmentsQueue: AttachmentsQueue, + private val userId: String?, +) : ViewModel() { private val _inputText = MutableStateFlow("") val inputText: StateFlow = _inputText private val _editingItem = MutableStateFlow(null) val editingItem: StateFlow = _editingItem - fun watchItems(listId: String?): Flow> { - return db.watch(""" - SELECT * - FROM $TODOS_TABLE - WHERE list_id = ? - ORDER by id + fun watchItems(listId: String?): Flow> = + db.watch( + """ + SELECT + t.*, a.local_uri + FROM + $TODOS_TABLE t + LEFT JOIN attachments a ON t.photo_id = a.id + WHERE + t.list_id = ? + ORDER BY t.id; """, - if(listId != null) listOf(listId) else null + if (listId != null) listOf(listId) else null, ) { cursor -> TodoItem( id = cursor.getString("id"), @@ -42,45 +52,54 @@ internal class Todo( completedBy = cursor.getStringOptional("completed_by"), completed = cursor.getLongOptional("completed") == 1L, listId = cursor.getString("list_id"), - photoId = cursor.getStringOptional("photo_id") + photoId = cursor.getStringOptional("photo_id"), + photoURI = cursor.getStringOptional("local_uri") ) } - } fun onItemClicked(item: TodoItem) { _editingItem.value = item } - fun onItemDoneChanged(item: TodoItem, isDone: Boolean) { + fun onItemDoneChanged( + item: TodoItem, + isDone: Boolean, + ) { updateItem(item = item) { it.copy( completed = isDone, - completedBy = if(isDone) userId else null, - completedAt = if(isDone) Clock.System.now().toString() else null + completedBy = if (isDone) userId else null, + completedAt = if (isDone) Clock.System.now().toString() else null, ) } } fun onItemDeleteClicked(item: TodoItem) { viewModelScope.launch { + if (item.photoId != null) { + attachmentsQueue.deleteFile(item.photoId) + } db.writeTransaction { tx -> tx.execute("DELETE FROM $TODOS_TABLE WHERE id = ?", listOf(item.id)) } } } - fun onAddItemClicked(userId: String?, listId: String?) { + fun onAddItemClicked( + userId: String?, + listId: String?, + ) { if (_inputText.value.isBlank()) return - if(userId == null || listId == null) { + if (userId == null || listId == null) { throw Exception("userId or listId is null") } viewModelScope.launch { - db.writeTransaction { tx -> + db.writeTransaction { tx -> tx.execute( "INSERT INTO $TODOS_TABLE (id, created_at, created_by, description, list_id) VALUES (uuid(), datetime(), ?, ?, ?)", - listOf(userId, _inputText.value, listId) + listOf(userId, _inputText.value, listId), ) } _inputText.value = "" @@ -106,24 +125,67 @@ internal class Todo( updateEditingItem(item = requireNotNull(_editingItem.value)) { it.copy( completed = isDone, - completedBy = if(isDone) userId else null, - completedAt = if(isDone) Clock.System.now().toString() else null + completedBy = if (isDone) userId else null, + completedAt = if (isDone) Clock.System.now().toString() else null, ) } } - private fun updateEditingItem(item: TodoItem, transformer: (item: TodoItem) -> TodoItem) { + fun onPhotoCapture(cameraService: CameraService) { + viewModelScope.launch { + val item = requireNotNull(_editingItem.value) + val photoData = try { + cameraService.takePicture() + } catch (ex: Exception) { + if (ex is CancellationException) { + throw ex + } else { + // otherwise ignore + return@launch + } + } + val attachment = attachmentsQueue.saveFile(data = flowOf(photoData), mediaType = "image/jped", fileExtension = "jpg" ) {tx, attachment -> + tx.execute("UPDATE $TODOS_TABLE SET photo_id = ? WHERE id = ?", listOf(attachment.id, item.id)) + } + + updateEditingItem(item = item) {it.copy(photoURI = attachment.localUri)} + } + } + + fun onPhotoDelete() { + viewModelScope.launch { + val item = requireNotNull(_editingItem.value) + attachmentsQueue.deleteFile(item.photoId!!) {tx, _ -> + tx.execute("UPDATE $TODOS_TABLE SET photo_id = NULL WHERE id = ?", listOf(item.id)) + } + updateEditingItem(item = item) {it.copy(photoURI = null)} + } + } + + private fun updateEditingItem( + item: TodoItem, + transformer: (item: TodoItem) -> TodoItem, + ) { _editingItem.value = transformer(item) } - private fun updateItem(item: TodoItem, transformer: (item: TodoItem) -> TodoItem) { + private fun updateItem( + item: TodoItem, + transformer: (item: TodoItem) -> TodoItem, + ) { viewModelScope.launch { val updatedItem = transformer(item) Logger.i("Updating item: $updatedItem") db.writeTransaction { tx -> tx.execute( "UPDATE $TODOS_TABLE SET description = ?, completed = ?, completed_by = ?, completed_at = ? WHERE id = ?", - listOf(updatedItem.description, updatedItem.completed, updatedItem.completedBy, updatedItem.completedAt, item.id) + listOf( + updatedItem.description, + updatedItem.completed, + updatedItem.completedBy, + updatedItem.completedAt, + item.id, + ), ) } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/screens/TodosScreen.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/screens/TodosScreen.kt index be261e26..403adfac 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/screens/TodosScreen.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/screens/TodosScreen.kt @@ -36,7 +36,7 @@ internal fun TodosScreen( onItemDoneChanged: (item: TodoItem, isDone: Boolean) -> Unit, onItemDeleteClicked: (item: TodoItem) -> Unit, onAddItemClicked: () -> Unit, - onInputTextChanged: (value: String) -> Unit, + onInputTextChanged: (value: String) -> Unit ) { Column(modifier) { TopAppBar( diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/ui/CameraService.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/ui/CameraService.kt new file mode 100644 index 00000000..2a78e23a --- /dev/null +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/ui/CameraService.kt @@ -0,0 +1,52 @@ +package com.powersync.androidexample.ui + +import android.net.Uri +import android.os.Environment +import androidx.activity.result.contract.ActivityResultContracts +import androidx.core.content.FileProvider +import com.powersync.androidexample.MainActivity +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.io.File +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale + +/** + * A very basic camera service. This should not be used in production. + */ +class CameraService(val activity: MainActivity) { + private var currentPhotoUri: Uri? = null + private var pictureResult: CompletableDeferred? = null + private var file: File? = null + private val mutex = Mutex() + + private val takePictureLauncher = activity.registerForActivityResult(ActivityResultContracts.TakePicture()) { success -> + if (success && currentPhotoUri != null) { + activity.contentResolver.openInputStream(currentPhotoUri!!)?.use { + pictureResult!!.complete(it.readBytes()) + } + file!!.delete() + + } else { + pictureResult!!.completeExceptionally(Exception("Could not capture photo")) + } + + file = null + currentPhotoUri = null + pictureResult = null + } + + suspend fun takePicture(): ByteArray = mutex.withLock { + pictureResult = CompletableDeferred() + val timeStamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()).format(Date()) + val storageDir = activity.getExternalFilesDir(Environment.DIRECTORY_PICTURES) + file = File.createTempFile("JPEG_${timeStamp}_", ".jpg", storageDir) + currentPhotoUri = FileProvider.getUriForFile(activity, "${activity.packageName}.fileprovider", file!!) + + takePictureLauncher.launch(currentPhotoUri!!) + + return pictureResult!!.await() + } +} \ No newline at end of file diff --git a/demos/android-supabase-todolist/app/src/main/res/xml/filepaths.xml b/demos/android-supabase-todolist/app/src/main/res/xml/filepaths.xml new file mode 100644 index 00000000..6f554b86 --- /dev/null +++ b/demos/android-supabase-todolist/app/src/main/res/xml/filepaths.xml @@ -0,0 +1,4 @@ + + + + diff --git a/demos/android-supabase-todolist/app/src/main/res/xml/network_security_config.xml b/demos/android-supabase-todolist/app/src/main/res/xml/network_security_config.xml new file mode 100644 index 00000000..de61259a --- /dev/null +++ b/demos/android-supabase-todolist/app/src/main/res/xml/network_security_config.xml @@ -0,0 +1,6 @@ + + + + localhost + + diff --git a/demos/android-supabase-todolist/gradle/libs.versions.toml b/demos/android-supabase-todolist/gradle/libs.versions.toml index 40dd839b..4268a9db 100644 --- a/demos/android-supabase-todolist/gradle/libs.versions.toml +++ b/demos/android-supabase-todolist/gradle/libs.versions.toml @@ -12,7 +12,7 @@ composeBom = "2025.02.00" materialIconsExtended = "1.7.8" uuid = "0.8.2" kermit = "2.0.5" -sqldelight= "2.0.2" +sqldelight = "2.0.2" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } diff --git a/demos/android-supabase-todolist/local.properties.example b/demos/android-supabase-todolist/local.properties.example index 0b8f0d04..dd05ddce 100644 --- a/demos/android-supabase-todolist/local.properties.example +++ b/demos/android-supabase-todolist/local.properties.example @@ -11,6 +11,7 @@ sdk.dir=/Users/dominic/Library/Android/sdk SUPABASE_URL=https://foo.supabase.co SUPABASE_ANON_KEY=foo +SUPABASE_ATTACHMENT_BUCKET=media # optional attachment bucket POWERSYNC_URL=https://foo.powersync.journeyapps.com # Set to true to use released PowerSync packages instead of the ones built locally. USE_RELEASED_POWERSYNC_VERSIONS=false diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5f1b0146..2e4aff1a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -100,7 +100,7 @@ sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } stately-concurrency = { module = "co.touchlab:stately-concurrency", version.ref = "stately" } supabase-client = { module = "io.github.jan-tennert.supabase:postgrest-kt", version.ref = "supabase" } supabase-auth = { module = "io.github.jan-tennert.supabase:auth-kt", version.ref = "supabase" } - +supabase-storage = {module = "io.github.jan-tennert.supabase:storage-kt", version.ref = "supabase"} androidx-sqliteFramework = { module = "androidx.sqlite:sqlite-framework", version.ref = "androidxSqlite" } # Sample - Android From f414eb26bac74a806e7866a7a3bdba6b3897193f Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 4 Apr 2025 18:11:11 +0200 Subject: [PATCH 13/51] Switch to open class. --- .../AbstractLocalStorageAdapter.android.kt | 12 -- .../kotlin/com/powersync/AttachmentsTest.kt | 47 +++++-- .../testutils/TestAttachmentsQueue.kt | 41 ------ ...tAttachmentQueue.kt => AttachmentQueue.kt} | 117 ++++++++---------- .../attachments/LocalStorageAdapter.kt | 2 - .../storage/AbstractLocalStorageAdapter.kt | 8 -- .../storage/IOLocalStorageAdapter.kt | 27 ++-- .../AbstractLocalStorageAdapter.ios.kt | 9 -- .../AbstractLocalStorageAdapter.jvm.kt | 8 -- .../java/com/powersync/androidexample/App.kt | 18 ++- .../androidexample/AttachmentsQueue.kt | 23 ---- .../java/com/powersync/androidexample/Auth.kt | 4 +- .../powersync/androidexample/MainActivity.kt | 14 +-- .../androidexample/powersync/Todo.kt | 4 +- 14 files changed, 122 insertions(+), 212 deletions(-) delete mode 100644 core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt delete mode 100644 core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt rename core/src/commonMain/kotlin/com/powersync/attachments/{AbstractAttachmentQueue.kt => AttachmentQueue.kt} (86%) delete mode 100644 core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt delete mode 100644 core/src/iosMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.ios.kt delete mode 100644 core/src/jvmMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.jvm.kt delete mode 100644 demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/AttachmentsQueue.kt diff --git a/core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt b/core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt deleted file mode 100644 index f890caec..00000000 --- a/core/src/androidMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.android.kt +++ /dev/null @@ -1,12 +0,0 @@ -package com.powersync.attachments.storage - -import android.os.Environment -import android.os.Environment.DIRECTORY_DOCUMENTS -import com.powersync.attachments.LocalStorageAdapter - -import okio.FileSystem - -@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") -public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { - actual override fun getUserStorageDirectory(): String = FileSystem.SYSTEM_TEMPORARY_DIRECTORY.normalized().toString() -} diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 4136fb39..5419e5b9 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -3,14 +3,17 @@ package com.powersync import app.cash.turbine.turbineScope import co.touchlab.kermit.ExperimentalKermitApi import com.powersync.attachments.Attachment +import com.powersync.attachments.AttachmentQueue import com.powersync.attachments.AttachmentState import com.powersync.attachments.RemoteStorageAdapter import com.powersync.attachments.SyncErrorHandler +import com.powersync.attachments.WatchedAttachmentItem import com.powersync.attachments.createAttachmentsTable +import com.powersync.db.getString import com.powersync.db.schema.Schema import com.powersync.testutils.MockedRemoteStorage -import com.powersync.testutils.TestAttachmentsQueue import com.powersync.testutils.UserRow +import com.powersync.testutils.getTempDir import dev.mokkery.answering.throws import dev.mokkery.everySuspend import dev.mokkery.matcher.ArgMatchersScope @@ -63,6 +66,24 @@ class AttachmentsTest { .cleanup("testdb") } + fun watchAttachments() = + database.watch( + sql = + """ + SELECT + photo_id + FROM + users + WHERE + photo_id IS NOT NULL + """, + ) { + WatchedAttachmentItem( + id = it.getString("photo_id"), + fileExtension = "jpg", + ) + } + @Test fun testAttachmentDownload() = runTest(timeout = 5.minutes) { @@ -70,9 +91,11 @@ class AttachmentsTest { val remote = spy(MockedRemoteStorage()) val queue = - TestAttachmentsQueue( + AttachmentQueue( db = database, remoteStorage = remote, + attachmentDirectory = getTempDir(), + watchedAttachments = watchAttachments(), /** * Sets the cache limit to zero for this test. Archived records will * immediately be deleted. @@ -174,9 +197,11 @@ class AttachmentsTest { val remote = spy(MockedRemoteStorage()) val queue = - TestAttachmentsQueue( + AttachmentQueue( db = database, remoteStorage = remote, + attachmentDirectory = getTempDir(), + watchedAttachments = watchAttachments(), /** * Sets the cache limit to zero for this test. Archived records will * immediately be deleted. @@ -246,10 +271,10 @@ class AttachmentsTest { // Now clear the user's photo_id. The attachment should be archived database.execute( """ - UPDATE - users - SET - photo_id = NULL + UPDATE + users + SET + photo_id = NULL """, ) @@ -276,9 +301,11 @@ class AttachmentsTest { val remote = spy(MockedRemoteStorage()) val queue = - TestAttachmentsQueue( + AttachmentQueue( db = database, remoteStorage = remote, + attachmentDirectory = getTempDir(), + watchedAttachments = watchAttachments(), /** * Keep some items in the cache */ @@ -388,9 +415,11 @@ class AttachmentsTest { } val queue = - TestAttachmentsQueue( + AttachmentQueue( db = database, remoteStorage = remote, + attachmentDirectory = getTempDir(), + watchedAttachments = watchAttachments(), archivedCacheLimit = 0, errorHandler = object : SyncErrorHandler { diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt deleted file mode 100644 index 40473759..00000000 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestAttachmentsQueue.kt +++ /dev/null @@ -1,41 +0,0 @@ -package com.powersync.testutils - -import com.powersync.PowerSyncDatabase -import com.powersync.attachments.AbstractAttachmentQueue -import com.powersync.attachments.RemoteStorageAdapter -import com.powersync.attachments.SyncErrorHandler -import com.powersync.attachments.WatchedAttachmentItem -import com.powersync.db.getString -import kotlinx.coroutines.flow.Flow - -internal class TestAttachmentsQueue( - db: PowerSyncDatabase, - remoteStorage: RemoteStorageAdapter, - archivedCacheLimit: Long, - errorHandler: SyncErrorHandler? = null, -) : AbstractAttachmentQueue( - db, - remoteStorage, - archivedCacheLimit = archivedCacheLimit, - errorHandler = errorHandler, - ) { - override fun watchAttachments(): Flow> = - db.watch( - sql = - """ - SELECT - photo_id - FROM - users - WHERE - photo_id IS NOT NULL - """, - ) { - WatchedAttachmentItem(id = it.getString("photo_id"), fileExtension = "jpg") - } - - /** - * For tests this uses a temporary directory. On iOS it uses the user storage directory - */ - override fun getStorageDirectory(): String = getTempDir() -} diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt similarity index 86% rename from core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 8474443c..18c5d490 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AbstractAttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -46,12 +46,12 @@ public data class WatchedAttachmentItem( } /** - * Abstract class used to implement the attachment queue + * Class used to implement the attachment queue * Requires a PowerSyncDatabase, an implementation of * AbstractRemoteStorageAdapter and an attachment directory name which will * determine which folder attachments are stored into. */ -public abstract class AbstractAttachmentQueue( +public open class AttachmentQueue( /** * PowerSync database client */ @@ -61,14 +61,36 @@ public abstract class AbstractAttachmentQueue( */ public val remoteStorage: RemoteStorageAdapter, /** - * Provides access to local filesystem storage methods + * Directory name where attachment files will be written to disk. + * This will be created if it does not exist */ - public val localStorage: LocalStorageAdapter = IOLocalStorageAdapter(), + private val attachmentDirectory: String, /** - * Directory name where attachment files will be written to disk. - * This will be created under the directory returned from [getStorageDirectory] + * A flow for the current state of local attachments + * ```kotlin + * watchedAttachment = db.watch( + * sql = + * """ + * SELECT + * photo_id as id + * FROM + * checklists + * WHERE + * photo_id IS NOT NULL + * """, + * ) { cursor -> + * WatchedAttachmentItem( + * id = cursor.getString("id"), + * fileExtension = "jpg", + * ) + * } + * ``` + */ + private val watchedAttachments: Flow>, + /** + * Provides access to local filesystem storage methods */ - private val attachmentDirectoryName: String = DEFAULT_ATTACHMENTS_DIRECTORY_NAME, + public val localStorage: LocalStorageAdapter = IOLocalStorageAdapter(), /** * SQLite table where attachment state will be recorded */ @@ -159,10 +181,10 @@ public abstract class AbstractAttachmentQueue( throw Exception("Attachment queue has been closed") } // Ensure the directory where attachments are downloaded, exists - localStorage.makeDir(getStorageDirectory()) + localStorage.makeDir(attachmentDirectory) subdirectories?.forEach { subdirectory -> - localStorage.makeDir(Path(getStorageDirectory(), subdirectory).toString()) + localStorage.makeDir(Path(attachmentDirectory, subdirectory).toString()) } val scope = CoroutineScope(Dispatchers.IO) @@ -186,7 +208,7 @@ public abstract class AbstractAttachmentQueue( val watchJob = launch { // Watch local attachment relationships and sync the attachment records - watchAttachments().collect { items -> + watchedAttachments.collect { items -> processWatchedAttachments(items) } } @@ -213,31 +235,6 @@ public abstract class AbstractAttachmentQueue( } } - /** - * Creates a watcher for the current state of local attachments - * ```kotlin - * public fun watchAttachments(): Flow> = - * db.watch( - * sql = - * """ - * SELECT - * photo_id as id - * FROM - * checklists - * WHERE - * photo_id IS NOT NULL - * """, - * ) { cursor -> - * WatchedAttachmentItem( - * id = cursor.getString("id"), - * fileExtension = "jpg", - * ) - * } - * ``` - */ - @Throws(PowerSyncException::class) - public abstract fun watchAttachments(): Flow> - /** * Resolves the filename for new attachment items. * A new attachment from [watchAttachments] might not include a filename. @@ -374,18 +371,19 @@ public abstract class AbstractAttachmentQueue( * assignment should happen in the same transaction. */ db.writeTransaction { tx -> - val attachment = Attachment( - id = id, - filename = filename, - size = fileSize, - mediaType = mediaType, - state = AttachmentState.QUEUED_UPLOAD.ordinal, - localUri = localUri, - ) + val attachment = + Attachment( + id = id, + filename = filename, + size = fileSize, + mediaType = mediaType, + state = AttachmentState.QUEUED_UPLOAD.ordinal, + localUri = localUri, + ) /** * Allow consumers to set relationships to this attachment id - */ + */ updateHook?.invoke(tx, attachment) return@writeTransaction attachmentsService.upsertAttachment( @@ -403,9 +401,10 @@ public abstract class AbstractAttachmentQueue( * This method can be overriden for custom behaviour. */ @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun deleteFile(attachmentId: String, - updateHook: ((context: ConnectionContext, attachment: Attachment) -> Unit)? = null, - ): Attachment = + public open suspend fun deleteFile( + attachmentId: String, + updateHook: ((context: ConnectionContext, attachment: Attachment) -> Unit)? = null, + ): Attachment = runWrappedSuspending { val attachment = attachmentsService.getAttachment(attachmentId) @@ -420,29 +419,11 @@ public abstract class AbstractAttachmentQueue( } } - /** - * Returns the local file path for the given filename, used to store in the database. - * Example: filename: "attachment-1.jpg" returns "attachments/attachment-1.jpg" - */ - public open fun getLocalFilePathSuffix(filename: String): String = Path(attachmentDirectoryName, filename).toString() - - /** - * Returns the directory where attachments are stored on the device, used to make dir - * Example: "/data/user/0/com.yourdomain.app/files/attachments/" - */ - public open fun getStorageDirectory(): String { - val userStorageDirectory = localStorage.getUserStorageDirectory() - return Path(userStorageDirectory, attachmentDirectoryName).toString() - } - /** * Return users storage directory with the attachmentPath use to load the file. - * Example: filePath: "attachments/attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg" + * Example: filePath: "attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg" */ - public open fun getLocalUri(filename: String): String { - val storageDirectory = getStorageDirectory() - return Path(storageDirectory, filename).toString() - } + public open fun getLocalUri(filename: String): String = Path(attachmentDirectory, filename).toString() /** * Removes all archived items @@ -460,6 +441,6 @@ public abstract class AbstractAttachmentQueue( public suspend fun clearQueue() { attachmentsService.clearQueue() // Remove the attachments directory - localStorage.rmDir(getStorageDirectory()) + localStorage.rmDir(attachmentDirectory) } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt index c7eb53a1..00662250 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt @@ -41,6 +41,4 @@ public interface LocalStorageAdapter { sourcePath: String, targetPath: String, ): Unit - - public fun getUserStorageDirectory(): String } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt deleted file mode 100644 index 07cf77c1..00000000 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.kt +++ /dev/null @@ -1,8 +0,0 @@ -package com.powersync.attachments.storage - -import com.powersync.attachments.LocalStorageAdapter - -@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") -public expect abstract class AbstractLocalStorageAdapter() : LocalStorageAdapter { - override fun getUserStorageDirectory(): String -} diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt index 0db5109f..7a9ca4f7 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt @@ -1,6 +1,6 @@ package com.powersync.attachments.storage -import com.powersync.attachments.storage.AbstractLocalStorageAdapter +import com.powersync.attachments.LocalStorageAdapter import com.powersync.db.runWrappedSuspending import io.ktor.utils.io.core.readBytes import io.ktor.utils.io.core.remaining @@ -14,13 +14,11 @@ import kotlinx.io.Buffer import kotlinx.io.buffered import kotlinx.io.files.Path import kotlinx.io.files.SystemFileSystem -import kotlinx.io.readByteArray -import kotlin.math.min /** * Storage adapter for local storage using the KotlinX IO library */ -public class IOLocalStorageAdapter : AbstractLocalStorageAdapter() { +public class IOLocalStorageAdapter : LocalStorageAdapter { public override suspend fun saveFile( filePath: String, data: Flow, @@ -50,18 +48,17 @@ public class IOLocalStorageAdapter : AbstractLocalStorageAdapter() { mediaType: String?, ): Flow = flow { - SystemFileSystem.source(Path(filePath)).use { source -> - source.buffered().use { bufferedSource -> - var remaining = 0L - val bufferSize = 8192L - do { - bufferedSource.request(bufferSize) - remaining = bufferedSource.remaining - emit(bufferedSource.readBytes(remaining.toInt())) - } while (remaining > 0) - - } + SystemFileSystem.source(Path(filePath)).use { source -> + source.buffered().use { bufferedSource -> + var remaining = 0L + val bufferSize = 8192L + do { + bufferedSource.request(bufferSize) + remaining = bufferedSource.remaining + emit(bufferedSource.readBytes(remaining.toInt())) + } while (remaining > 0) } + } }.flowOn(Dispatchers.IO) public override suspend fun deleteFile(filePath: String): Unit = diff --git a/core/src/iosMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.ios.kt b/core/src/iosMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.ios.kt deleted file mode 100644 index 87094951..00000000 --- a/core/src/iosMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.ios.kt +++ /dev/null @@ -1,9 +0,0 @@ -package com.powersync.attachments.storage - -import com.powersync.attachments.LocalStorageAdapter -import platform.Foundation.NSHomeDirectory - -@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") -public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { - actual override fun getUserStorageDirectory(): String = NSHomeDirectory() -} diff --git a/core/src/jvmMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.jvm.kt b/core/src/jvmMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.jvm.kt deleted file mode 100644 index b4caa2a3..00000000 --- a/core/src/jvmMain/kotlin/com/powersync/attachments/storage/AbstractLocalStorageAdapter.jvm.kt +++ /dev/null @@ -1,8 +0,0 @@ -package com.powersync.attachments.storage - -import com.powersync.attachments.LocalStorageAdapter - -@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") -public actual abstract class AbstractLocalStorageAdapter : LocalStorageAdapter { - actual override fun getUserStorageDirectory(): String = System.getProperty("user.home") -} diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index 69c94f28..3f2e5a7d 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -12,12 +12,14 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.rememberUpdatedState import androidx.compose.ui.Modifier import com.powersync.PowerSyncDatabase -import com.powersync.androidexample.AttachmentsQueue import com.powersync.androidexample.BuildConfig import com.powersync.androidexample.SupabaseRemoteStorage import com.powersync.androidexample.ui.CameraService +import com.powersync.attachments.AttachmentQueue +import com.powersync.attachments.WatchedAttachmentItem import com.powersync.compose.rememberDatabaseDriverFactory import com.powersync.connector.supabase.SupabaseConnector +import com.powersync.db.getString import com.powersync.demos.components.EditDialog import com.powersync.demos.powersync.ListContent import com.powersync.demos.powersync.ListItem @@ -30,7 +32,7 @@ import com.powersync.demos.screens.TodosScreen import kotlinx.coroutines.runBlocking @Composable -fun App(cameraService: CameraService) { +fun App(cameraService: CameraService, attachmentDirectory: String) { val driverFactory = rememberDatabaseDriverFactory() val supabase = remember { @@ -44,7 +46,17 @@ fun App(cameraService: CameraService) { val db = remember { PowerSyncDatabase(driverFactory, schema, dbFilename = "222.sqlite") } val attachments = - remember { AttachmentsQueue(db = db, remoteStorage = SupabaseRemoteStorage(supabase)) } + remember { AttachmentQueue( + db = db, remoteStorage = SupabaseRemoteStorage(supabase), + attachmentDirectory = attachmentDirectory, + watchedAttachments = db.watch( + "SELECT photo_id from todos WHERE photo_id IS NOT NULL" + ) { + WatchedAttachmentItem( + id = it.getString("photo_id"), + fileExtension = "jpg" + ) + }) } val syncStatus = db.currentStatus val status by syncStatus.asFlow().collectAsState(syncStatus) diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/AttachmentsQueue.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/AttachmentsQueue.kt deleted file mode 100644 index b9827392..00000000 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/AttachmentsQueue.kt +++ /dev/null @@ -1,23 +0,0 @@ -package com.powersync.androidexample - -import com.powersync.PowerSyncDatabase -import com.powersync.attachments.AbstractAttachmentQueue -import com.powersync.attachments.RemoteStorageAdapter -import com.powersync.attachments.WatchedAttachmentItem -import com.powersync.db.getString -import kotlinx.coroutines.flow.Flow - -class AttachmentsQueue(db: PowerSyncDatabase, - remoteStorage: RemoteStorageAdapter -) : AbstractAttachmentQueue(db, remoteStorage, archivedCacheLimit = 0) { - override fun watchAttachments(): Flow> { - return db.watch( - "SELECT photo_id from todos WHERE photo_id IS NOT NULL" - ) { - WatchedAttachmentItem( - id = it.getString("photo_id"), - fileExtension = "jpg" - ) - } - } -} \ No newline at end of file diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt index 87633a74..7e711dc5 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt @@ -4,7 +4,7 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase -import com.powersync.androidexample.AttachmentsQueue +import com.powersync.attachments.AttachmentQueue import com.powersync.connector.supabase.SupabaseConnector import io.github.jan.supabase.auth.status.RefreshFailureCause import io.github.jan.supabase.auth.status.SessionStatus @@ -21,7 +21,7 @@ sealed class AuthState { internal class AuthViewModel( private val supabase: SupabaseConnector, private val db: PowerSyncDatabase, - private val attachmentsQueue: AttachmentsQueue, + private val attachmentsQueue: AttachmentQueue, private val navController: NavController, ) : ViewModel() { private val _authState = MutableStateFlow(AuthState.SignedOut) diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt index d253a3ac..b6a4373a 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt @@ -1,20 +1,11 @@ package com.powersync.androidexample -import android.net.Uri import android.os.Bundle -import android.os.Environment import androidx.activity.ComponentActivity import androidx.activity.compose.setContent -import androidx.activity.result.contract.ActivityResultContracts -import androidx.core.content.FileProvider import androidx.core.splashscreen.SplashScreen.Companion.installSplashScreen import com.powersync.androidexample.ui.CameraService import com.powersync.demos.App -import kotlinx.coroutines.CompletableDeferred -import java.io.File -import java.text.SimpleDateFormat -import java.util.Date -import java.util.Locale class MainActivity : ComponentActivity() { private val cameraService = CameraService(this) @@ -24,7 +15,10 @@ class MainActivity : ComponentActivity() { super.onCreate(savedInstanceState) setContent { - App(cameraService = cameraService) + App( + cameraService = cameraService, + attachmentDirectory = "${applicationContext.filesDir.canonicalPath}/attachments" + ) } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt index c6edae35..139c6193 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt @@ -5,8 +5,8 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase -import com.powersync.androidexample.AttachmentsQueue import com.powersync.androidexample.ui.CameraService +import com.powersync.attachments.AttachmentQueue import com.powersync.db.getLongOptional import com.powersync.db.getString import com.powersync.db.getStringOptional @@ -20,7 +20,7 @@ import kotlinx.datetime.Clock internal class Todo( private val db: PowerSyncDatabase, - private val attachmentsQueue: AttachmentsQueue, + private val attachmentsQueue: AttachmentQueue, private val userId: String?, ) : ViewModel() { private val _inputText = MutableStateFlow("") From 67eee584fcf44cdaeb908c418ce89800ad46204c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Sun, 6 Apr 2025 20:05:47 +0200 Subject: [PATCH 14/51] fix demo navigation state --- .../app/src/main/java/com/powersync/androidexample/App.kt | 7 +++++-- .../app/src/main/java/com/powersync/androidexample/Auth.kt | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index 3f2e5a7d..b376e8ab 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -44,7 +44,7 @@ fun App(cameraService: CameraService, attachmentDirectory: String) { ) } - val db = remember { PowerSyncDatabase(driverFactory, schema, dbFilename = "222.sqlite") } + val db = remember { PowerSyncDatabase(driverFactory, schema, dbFilename = "333.sqlite") } val attachments = remember { AttachmentQueue( db = db, remoteStorage = SupabaseRemoteStorage(supabase), @@ -61,7 +61,10 @@ fun App(cameraService: CameraService, attachmentDirectory: String) { val syncStatus = db.currentStatus val status by syncStatus.asFlow().collectAsState(syncStatus) - val navController = remember { NavController(Screen.Home) } + val navController = remember { + NavController(Screen.Home) + } + val authViewModel = remember { AuthViewModel(supabase, db, attachments, navController) diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt index 7e711dc5..7f06496b 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt @@ -6,6 +6,7 @@ import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase import com.powersync.attachments.AttachmentQueue import com.powersync.connector.supabase.SupabaseConnector +import com.powersync.demos.screens.SignInScreen import io.github.jan.supabase.auth.status.RefreshFailureCause import io.github.jan.supabase.auth.status.SessionStatus import kotlinx.coroutines.flow.MutableStateFlow @@ -38,7 +39,10 @@ internal class AuthViewModel( _userId.value = it.session.user?.id db.connect(supabase) attachmentsQueue.startSync() - navController.navigate(Screen.Home) + if (navController.currentScreen.value is Screen.SignIn + || navController.currentScreen.value is Screen.SignUp) { + navController.navigate(Screen.Home) + } } is SessionStatus.Initializing -> Logger.e("Loading from storage") is SessionStatus.RefreshFailure -> { From d00f0b18bcf590e14dd5edc82f101730ccd6f1f5 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 7 Apr 2025 11:36:27 +0200 Subject: [PATCH 15/51] improve locking to avoid rare race conditions --- .../kotlin/com/powersync/AttachmentsTest.kt | 10 +- .../testutils/MockedRemoteStorage.kt | 4 +- .../powersync/attachments/AttachmentQueue.kt | 144 +++++++------ .../attachments/AttachmentService.kt | 204 +++--------------- ...LocalStorageAdapter.kt => LocalStorage.kt} | 4 +- ...moteStorageAdapter.kt => RemoteStorage.kt} | 2 +- .../implementation/AttachmentContextImpl.kt | 192 +++++++++++++++++ .../implementation/AttachmentServiceImpl.kt | 97 +++++++++ .../storage/IOLocalStorageAdapter.kt | 4 +- .../attachments/sync/SyncingService.kt | 54 ++--- 10 files changed, 438 insertions(+), 277 deletions(-) rename core/src/commonMain/kotlin/com/powersync/attachments/{LocalStorageAdapter.kt => LocalStorage.kt} (94%) rename core/src/commonMain/kotlin/com/powersync/attachments/{RemoteStorageAdapter.kt => RemoteStorage.kt} (93%) create mode 100644 core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt create mode 100644 core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 5419e5b9..19a15db9 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -5,7 +5,7 @@ import co.touchlab.kermit.ExperimentalKermitApi import com.powersync.attachments.Attachment import com.powersync.attachments.AttachmentQueue import com.powersync.attachments.AttachmentState -import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.RemoteStorage import com.powersync.attachments.SyncErrorHandler import com.powersync.attachments.WatchedAttachmentItem import com.powersync.attachments.createAttachmentsTable @@ -88,7 +88,7 @@ class AttachmentsTest { fun testAttachmentDownload() = runTest(timeout = 5.minutes) { turbineScope(timeout = 5.minutes) { - val remote = spy(MockedRemoteStorage()) + val remote = spy(MockedRemoteStorage()) val queue = AttachmentQueue( @@ -194,7 +194,7 @@ class AttachmentsTest { fun testAttachmentUpload() = runTest { turbineScope { - val remote = spy(MockedRemoteStorage()) + val remote = spy(MockedRemoteStorage()) val queue = AttachmentQueue( @@ -298,7 +298,7 @@ class AttachmentsTest { fun testAttachmentCachedDownload() = runTest { turbineScope { - val remote = spy(MockedRemoteStorage()) + val remote = spy(MockedRemoteStorage()) val queue = AttachmentQueue( @@ -410,7 +410,7 @@ class AttachmentsTest { runTest { turbineScope { val remote = - mock { + mock { everySuspend { downloadFile(any()) } throws (Exception("Test error")) } diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt index 6d987509..8baa0ed0 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/MockedRemoteStorage.kt @@ -1,11 +1,11 @@ package com.powersync.testutils import com.powersync.attachments.Attachment -import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.RemoteStorage import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow -class MockedRemoteStorage : RemoteStorageAdapter { +class MockedRemoteStorage : RemoteStorage { override suspend fun uploadFile( fileData: Flow, attachment: Attachment, diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 18c5d490..37e4a8d8 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -3,6 +3,7 @@ package com.powersync.attachments import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase import com.powersync.PowerSyncException +import com.powersync.attachments.implementation.AttachmentServiceImpl import com.powersync.attachments.storage.IOLocalStorageAdapter import com.powersync.attachments.sync.SyncingService import com.powersync.db.internal.ConnectionContext @@ -59,7 +60,7 @@ public open class AttachmentQueue( /** * Adapter which interfaces with the remote storage backend */ - public val remoteStorage: RemoteStorageAdapter, + public val remoteStorage: RemoteStorage, /** * Directory name where attachment files will be written to disk. * This will be created if it does not exist @@ -90,7 +91,7 @@ public open class AttachmentQueue( /** * Provides access to local filesystem storage methods */ - public val localStorage: LocalStorageAdapter = IOLocalStorageAdapter(), + public val localStorage: LocalStorage = IOLocalStorageAdapter(), /** * SQLite table where attachment state will be recorded */ @@ -139,7 +140,7 @@ public open class AttachmentQueue( * - Create new attachment records for upload/download */ public val attachmentsService: AttachmentService = - AttachmentService( + AttachmentServiceImpl( db, attachmentsQueueTableName, logger, @@ -265,77 +266,82 @@ public open class AttachmentQueue( public open suspend fun processWatchedAttachments(items: List): Unit = runWrappedSuspending { /** - * Need to get all the attachments which are tracked in the DB. - * We might need to restore an archived attachment. + * Use a lock here to prevent conflicting state updates */ - val currentAttachments = attachmentsService.getAttachments() - val attachmentUpdates = mutableListOf() - - for (item in items) { - val existingQueueItem = currentAttachments.find { it.id == item.id } - - if (existingQueueItem == null) { - if (!downloadAttachments) { - continue - } - // This item should be added to the queue - // This item is assumed to be coming from an upstream sync - // Locally created new items should be persisted using [saveFile] before - // this point. - val filename = - resolveNewAttachmentFilename( - attachmentId = item.id, - fileExtension = item.fileExtension, - ) + attachmentsService.withLock { attachmentsContext -> + /** + * Need to get all the attachments which are tracked in the DB. + * We might need to restore an archived attachment. + */ + val currentAttachments = attachmentsContext.getAttachments() + val attachmentUpdates = mutableListOf() + + for (item in items) { + val existingQueueItem = currentAttachments.find { it.id == item.id } + + if (existingQueueItem == null) { + if (!downloadAttachments) { + continue + } + // This item should be added to the queue + // This item is assumed to be coming from an upstream sync + // Locally created new items should be persisted using [saveFile] before + // this point. + val filename = + item.filename ?: resolveNewAttachmentFilename( + attachmentId = item.id, + fileExtension = item.fileExtension, + ) - attachmentUpdates.add( - Attachment( - id = item.id, - filename = filename, - state = AttachmentState.QUEUED_DOWNLOAD.ordinal, - ), - ) - } else if - (existingQueueItem.state == AttachmentState.ARCHIVED.ordinal) { - // The attachment is present again. Need to queue it for sync. - // We might be able to optimize this in future - if (existingQueueItem.hasSynced == 1) { - // No remote action required, we can restore the record (avoids deletion) - attachmentUpdates.add( - existingQueueItem.copy(state = AttachmentState.SYNCED.ordinal), - ) - } else { - /** - * The localURI should be set if the record was meant to be downloaded - * and has been synced. If it's missing and hasSynced is false then - * it must be an upload operation - */ attachmentUpdates.add( - existingQueueItem.copy( - state = - if (existingQueueItem.localUri == null) { - AttachmentState.QUEUED_DOWNLOAD.ordinal - } else { - AttachmentState.QUEUED_UPLOAD.ordinal - }, + Attachment( + id = item.id, + filename = filename, + state = AttachmentState.QUEUED_DOWNLOAD.ordinal, ), ) + } else if + (existingQueueItem.state == AttachmentState.ARCHIVED.ordinal) { + // The attachment is present again. Need to queue it for sync. + // We might be able to optimize this in future + if (existingQueueItem.hasSynced == 1) { + // No remote action required, we can restore the record (avoids deletion) + attachmentUpdates.add( + existingQueueItem.copy(state = AttachmentState.SYNCED.ordinal), + ) + } else { + /** + * The localURI should be set if the record was meant to be downloaded + * and has been synced. If it's missing and hasSynced is false then + * it must be an upload operation + */ + attachmentUpdates.add( + existingQueueItem.copy( + state = + if (existingQueueItem.localUri == null) { + AttachmentState.QUEUED_DOWNLOAD.ordinal + } else { + AttachmentState.QUEUED_UPLOAD.ordinal + }, + ), + ) + } } } - } - /** - * Archive any items not specified in the watched items except for items pending delete. - */ - currentAttachments - .filter { - it.state != AttachmentState.QUEUED_DELETE.ordinal && - null == items.find { update -> update.id == it.id } - }.forEach { - attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED.ordinal)) - } + /** + * Archive any items not specified in the watched items except for items pending delete. + */ + currentAttachments + .filter { + it.state != AttachmentState.QUEUED_DELETE.ordinal && + null == items.find { update -> update.id == it.id } + }.forEach { + attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED.ordinal)) + } - attachmentsService.saveAttachments(attachmentUpdates) + attachmentsContext.saveAttachments(attachmentUpdates) + } } /** @@ -430,9 +436,11 @@ public open class AttachmentQueue( */ public suspend fun expireCache() { var done: Boolean - do { - done = syncingService.deleteArchivedAttachments() - } while (!done) + attachmentsService.withLock { context -> + do { + done = syncingService.deleteArchivedAttachments(context) + } while (!done) + } } /** diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt index 74a84f7c..fe09bd0b 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt @@ -1,198 +1,65 @@ package com.powersync.attachments -import co.touchlab.kermit.Logger -import com.powersync.PowerSyncDatabase import com.powersync.db.internal.ConnectionContext import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.map -import kotlinx.datetime.Clock -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json /** - * Service for interacting with the local attachment records. + * Context for performing Attachment operations. + * This typically is provided through a locking/exclusivity method. */ -public class AttachmentService( - private val db: PowerSyncDatabase, - private val tableName: String, - private val logger: Logger, - private val maxArchivedCount: Long, -) { - /** - * Table used for storing attachments in the attachment queue. - */ - private val table: String - get() = tableName - +public interface AttachmentContext { /** * Delete the attachment from the attachment queue. */ - public suspend fun deleteAttachment(id: String) { - db.execute("DELETE FROM $table WHERE id = ?", listOf(id)) - } + public suspend fun deleteAttachment(id: String): Unit /** * Set the state of the attachment to ignore. */ - public suspend fun ignoreAttachment(id: String) { - db.execute( - "UPDATE $table SET state = ? WHERE id = ?", - listOf(AttachmentState.ARCHIVED.ordinal, id), - ) - } + public suspend fun ignoreAttachment(id: String): Unit /** * Get the attachment from the attachment queue using an ID. */ - public suspend fun getAttachment(id: String): Attachment? = - db.getOptional("SELECT * FROM $table WHERE id = ?", listOf(id)) { - Attachment.fromCursor(it) - } + public suspend fun getAttachment(id: String): Attachment? /** * Save the attachment to the attachment queue. */ - public suspend fun saveAttachment(attachment: Attachment): Attachment = - db.writeLock { ctx -> - upsertAttachment(attachment, ctx) - } + public suspend fun saveAttachment(attachment: Attachment): Attachment /** * Save the attachments to the attachment queue. */ - public suspend fun saveAttachments(attachments: List) { - if (attachments.isEmpty()) { - return - } - - db.writeTransaction { tx -> - for (attachment in attachments) { - upsertAttachment(attachment, tx) - } - } - } + public suspend fun saveAttachments(attachments: List): Unit /** * Get all the ID's of attachments in the attachment queue. */ - public suspend fun getAttachmentIds(): List = - db.getAll( - "SELECT id FROM $table WHERE id IS NOT NULL", - ) { it.getString(0)!! } - - public suspend fun getAttachments(): List = - db.getAll( - """ - SELECT - * - FROM - $table - WHERE - id IS NOT NULL - ORDER BY - timestamp ASC - """, - ) { Attachment.fromCursor(it) } + public suspend fun getAttachmentIds(): List /** - * Gets all the active attachments which require an operation to be performed. + * Get all Attachment records present in the database. */ - public suspend fun getActiveAttachments(): List = - db.getAll( - """ - SELECT - * - FROM - $table - WHERE - state = ? - OR state = ? - OR state = ? - ORDER BY - timestamp ASC - """, - listOf( - AttachmentState.QUEUED_UPLOAD.ordinal, - AttachmentState.QUEUED_DOWNLOAD.ordinal, - AttachmentState.QUEUED_DELETE.ordinal, - ), - ) { Attachment.fromCursor(it) } + public suspend fun getAttachments(): List /** - * Watcher for changes to attachments table. - * Once a change is detected it will initiate a sync of the attachments + * Gets all the active attachments which require an operation to be performed. */ - public fun watchActiveAttachments(): Flow { - logger.i("Watching attachments...") - return db - .watch( - """ - SELECT - id - FROM - $table - WHERE - state = ? - OR state = ? - OR state = ? - ORDER BY - timestamp ASC - """, - listOf( - AttachmentState.QUEUED_UPLOAD.ordinal, - AttachmentState.QUEUED_DOWNLOAD.ordinal, - AttachmentState.QUEUED_DELETE.ordinal, - ), - ) { it.getString(0)!! } - // We only use changes here to trigger a sync consolidation - .map { Unit } - } + public suspend fun getActiveAttachments(): List /** * Helper function to clear the attachment queue * Currently only used for testing purposes. */ - public suspend fun clearQueue() { - logger.i("Clearing attachment queue...") - db.execute("DELETE FROM $table") - } + public suspend fun clearQueue(): Unit /** * Delete attachments which have been archived * @returns true if all items have been deleted. Returns false if there might be more archived * items remaining. */ - public suspend fun deleteArchivedAttachments(callback: suspend (attachments: List) -> Unit): Boolean { - // First fetch the attachments in order to allow other cleanup - val limit = 1000 - val attachments = - db.getAll( - """ - SELECT - * - FROM - $table - WHERE - state = ? - ORDER BY - timestamp DESC - LIMIT ? OFFSET ? - """, - listOf( - AttachmentState.ARCHIVED.ordinal, - limit, - maxArchivedCount, - ), - ) { Attachment.fromCursor(it) } - callback(attachments) - db.execute( - "DELETE FROM $table WHERE id IN (SELECT value FROM json_each(?));", - listOf( - Json.encodeToString(attachments.map { it.id }), - ), - ) - return attachments.size < limit - } + public suspend fun deleteArchivedAttachments(callback: suspend (attachments: List) -> Unit): Boolean /** * Upserts an attachment record synchronously given a database connection context. @@ -200,31 +67,22 @@ public class AttachmentService( public fun upsertAttachment( attachment: Attachment, context: ConnectionContext, - ): Attachment { - val updatedRecord = - attachment.copy( - timestamp = Clock.System.now().toEpochMilliseconds(), - ) + ): Attachment +} - context.execute( - """ - INSERT OR REPLACE INTO - $table (id, timestamp, filename, local_uri, media_type, size, state, has_synced) - VALUES - (?, ?, ?, ?, ?, ?, ?, ?) - """, - listOf( - updatedRecord.id, - updatedRecord.timestamp, - updatedRecord.filename, - updatedRecord.localUri, - updatedRecord.mediaType, - updatedRecord.size, - updatedRecord.state, - updatedRecord.hasSynced, - ), - ) +/** + * Service for interacting with the local attachment records. + */ +public interface AttachmentService : AttachmentContext { + /** + * Watcher for changes to attachments table. + * Once a change is detected it will initiate a sync of the attachments + */ + public fun watchActiveAttachments(): Flow - return attachment - } + /** + * Executes a callback with an exclusive lock on all attachment operations. + * This helps prevent race conditions between different updates. + */ + public suspend fun withLock(action: suspend (context: AttachmentContext) -> R): R } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorage.kt similarity index 94% rename from core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/LocalStorage.kt index 00662250..1ff6fef4 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorage.kt @@ -5,9 +5,9 @@ import kotlinx.coroutines.flow.Flow import kotlin.coroutines.cancellation.CancellationException /** - * Storage adapter for local storage + * Provides access to local storage on a device */ -public interface LocalStorageAdapter { +public interface LocalStorage { /** * Saves a source of data bytes to a path. * @returns the bytesize of the file diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorage.kt similarity index 93% rename from core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorageAdapter.kt rename to core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorage.kt index 54c3b9a8..8035d94e 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorage.kt @@ -5,7 +5,7 @@ import kotlinx.coroutines.flow.Flow /** * Adapter for interfacing with remote attachment storage. */ -public interface RemoteStorageAdapter { +public interface RemoteStorage { /** * Upload a file to remote storage */ diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt new file mode 100644 index 00000000..55d106a2 --- /dev/null +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt @@ -0,0 +1,192 @@ +package com.powersync.attachments.implementation + +import co.touchlab.kermit.Logger +import com.powersync.PowerSyncDatabase +import com.powersync.attachments.Attachment +import com.powersync.attachments.AttachmentContext +import com.powersync.attachments.AttachmentState +import com.powersync.db.internal.ConnectionContext +import kotlinx.datetime.Clock +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json + +public open class AttachmentContextImpl( + public val db: PowerSyncDatabase, + public val table: String, + private val logger: Logger, + private val maxArchivedCount: Long, +) : AttachmentContext { + /** + * Delete the attachment from the attachment queue. + */ + public override suspend fun deleteAttachment(id: String) { + db.execute("DELETE FROM $table WHERE id = ?", listOf(id)) + } + + /** + * Set the state of the attachment to ignore. + */ + public override suspend fun ignoreAttachment(id: String) { + db.execute( + "UPDATE $table SET state = ? WHERE id = ?", + listOf(AttachmentState.ARCHIVED.ordinal, id), + ) + } + + /** + * Get the attachment from the attachment queue using an ID. + */ + public override suspend fun getAttachment(id: String): Attachment? = + db.getOptional("SELECT * FROM $table WHERE id = ?", listOf(id)) { + Attachment.fromCursor(it) + } + + /** + * Save the attachment to the attachment queue. + */ + public override suspend fun saveAttachment(attachment: Attachment): Attachment = + db.writeLock { ctx -> + upsertAttachment(attachment, ctx) + } + + /** + * Save the attachments to the attachment queue. + */ + public override suspend fun saveAttachments(attachments: List) { + if (attachments.isEmpty()) { + return + } + + db.writeTransaction { tx -> + for (attachment in attachments) { + upsertAttachment(attachment, tx) + } + } + } + + /** + * Get all the ID's of attachments in the attachment queue. + */ + public override suspend fun getAttachmentIds(): List = + db.getAll( + "SELECT id FROM $table WHERE id IS NOT NULL", + ) { it.getString(0)!! } + + public override suspend fun getAttachments(): List = + db.getAll( + """ + SELECT + * + FROM + $table + WHERE + id IS NOT NULL + ORDER BY + timestamp ASC + """, + ) { Attachment.fromCursor(it) } + + /** + * Gets all the active attachments which require an operation to be performed. + */ + public override suspend fun getActiveAttachments(): List = + db.getAll( + """ + SELECT + * + FROM + $table + WHERE + state = ? + OR state = ? + OR state = ? + ORDER BY + timestamp ASC + """, + listOf( + AttachmentState.QUEUED_UPLOAD.ordinal, + AttachmentState.QUEUED_DOWNLOAD.ordinal, + AttachmentState.QUEUED_DELETE.ordinal, + ), + ) { Attachment.fromCursor(it) } + + /** + * Helper function to clear the attachment queue + * Currently only used for testing purposes. + */ + public override suspend fun clearQueue() { + logger.i("Clearing attachment queue...") + db.execute("DELETE FROM $table") + } + + /** + * Delete attachments which have been archived + * @returns true if all items have been deleted. Returns false if there might be more archived + * items remaining. + */ + public override suspend fun deleteArchivedAttachments(callback: suspend (attachments: List) -> Unit): Boolean { + // First fetch the attachments in order to allow other cleanup + val limit = 1000 + val attachments = + db.getAll( + """ + SELECT + * + FROM + $table + WHERE + state = ? + ORDER BY + timestamp DESC + LIMIT ? OFFSET ? + """, + listOf( + AttachmentState.ARCHIVED.ordinal, + limit, + maxArchivedCount, + ), + ) { Attachment.fromCursor(it) } + callback(attachments) + db.execute( + "DELETE FROM $table WHERE id IN (SELECT value FROM json_each(?));", + listOf( + Json.encodeToString(attachments.map { it.id }), + ), + ) + return attachments.size < limit + } + + /** + * Upserts an attachment record synchronously given a database connection context. + */ + public override fun upsertAttachment( + attachment: Attachment, + context: ConnectionContext, + ): Attachment { + val updatedRecord = + attachment.copy( + timestamp = Clock.System.now().toEpochMilliseconds(), + ) + + context.execute( + """ + INSERT OR REPLACE INTO + $table (id, timestamp, filename, local_uri, media_type, size, state, has_synced) + VALUES + (?, ?, ?, ?, ?, ?, ?, ?) + """, + listOf( + updatedRecord.id, + updatedRecord.timestamp, + updatedRecord.filename, + updatedRecord.localUri, + updatedRecord.mediaType, + updatedRecord.size, + updatedRecord.state, + updatedRecord.hasSynced, + ), + ) + + return attachment + } +} diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt new file mode 100644 index 00000000..79efab6f --- /dev/null +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt @@ -0,0 +1,97 @@ +package com.powersync.attachments.implementation + +import co.touchlab.kermit.Logger +import com.powersync.PowerSyncDatabase +import com.powersync.attachments.Attachment +import com.powersync.attachments.AttachmentContext +import com.powersync.attachments.AttachmentService +import com.powersync.attachments.AttachmentState +import com.powersync.db.internal.ConnectionContext +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * Service for interacting with the local attachment records. + */ +public open class AttachmentServiceImpl( + private val db: PowerSyncDatabase, + private val tableName: String, + private val logger: Logger, + private val maxArchivedCount: Long, +) : AttachmentService { + /** + * Table used for storing attachments in the attachment queue. + */ + private val table: String + get() = tableName + + private val mutex = Mutex() + + private val context: AttachmentContext = + AttachmentContextImpl( + db = db, + table = table, + logger = logger, + maxArchivedCount = maxArchivedCount, + ) + + public override suspend fun withLock(action: suspend (AttachmentContext) -> R): R = mutex.withLock { action(context) } + + /** + * Watcher for changes to attachments table. + * Once a change is detected it will initiate a sync of the attachments + */ + public override fun watchActiveAttachments(): Flow { + logger.i("Watching attachments...") + return db + .watch( + """ + SELECT + id + FROM + $table + WHERE + state = ? + OR state = ? + OR state = ? + ORDER BY + timestamp ASC + """, + listOf( + AttachmentState.QUEUED_UPLOAD.ordinal, + AttachmentState.QUEUED_DOWNLOAD.ordinal, + AttachmentState.QUEUED_DELETE.ordinal, + ), + ) { it.getString(0)!! } + // We only use changes here to trigger a sync consolidation + .map { Unit } + } + + override fun upsertAttachment( + attachment: Attachment, + context: ConnectionContext, + ): Attachment = this.context.upsertAttachment(attachment = attachment, context = context) + + override suspend fun deleteAttachment(id: String): Unit = withLock { it.deleteAttachment(id) } + + override suspend fun ignoreAttachment(id: String): Unit = withLock { it.ignoreAttachment(id) } + + override suspend fun getAttachment(id: String): Attachment? = withLock { it.getAttachment(id) } + + override suspend fun saveAttachment(attachment: Attachment): Attachment = withLock { it.saveAttachment(attachment) } + + override suspend fun saveAttachments(attachments: List): Unit = withLock { it.saveAttachments(attachments) } + + override suspend fun getAttachmentIds(): List = withLock { it.getAttachmentIds() } + + override suspend fun getAttachments(): List = withLock { it.getAttachments() } + + override suspend fun getActiveAttachments(): List = withLock { it.getActiveAttachments() } + + override suspend fun clearQueue(): Unit = withLock { it.clearQueue() } + + override suspend fun deleteArchivedAttachments(callback: suspend (List) -> Unit): Boolean = + withLock { it.deleteArchivedAttachments(callback) } +} diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt index 7a9ca4f7..a50035e1 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt @@ -1,6 +1,6 @@ package com.powersync.attachments.storage -import com.powersync.attachments.LocalStorageAdapter +import com.powersync.attachments.LocalStorage import com.powersync.db.runWrappedSuspending import io.ktor.utils.io.core.readBytes import io.ktor.utils.io.core.remaining @@ -18,7 +18,7 @@ import kotlinx.io.files.SystemFileSystem /** * Storage adapter for local storage using the KotlinX IO library */ -public class IOLocalStorageAdapter : LocalStorageAdapter { +public class IOLocalStorageAdapter : LocalStorage { public override suspend fun saveFile( filePath: String, data: Flow, diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index 183a8c61..d92f7c3d 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -3,10 +3,11 @@ package com.powersync.attachments.sync import co.touchlab.kermit.Logger import com.powersync.PowerSyncException import com.powersync.attachments.Attachment +import com.powersync.attachments.AttachmentContext import com.powersync.attachments.AttachmentService import com.powersync.attachments.AttachmentState -import com.powersync.attachments.LocalStorageAdapter -import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.LocalStorage +import com.powersync.attachments.RemoteStorage import com.powersync.attachments.SyncErrorHandler import com.powersync.utils.throttle import kotlinx.coroutines.CancellationException @@ -30,8 +31,8 @@ import kotlin.time.Duration.Companion.seconds * Service used to sync attachments between local and remote storage */ internal class SyncingService( - private val remoteStorage: RemoteStorageAdapter, - private val localStorage: LocalStorageAdapter, + private val remoteStorage: RemoteStorage, + private val localStorage: LocalStorage, private val attachmentsService: AttachmentService, private val getLocalUri: suspend (String) -> String, private val errorHandler: SyncErrorHandler?, @@ -65,24 +66,26 @@ internal class SyncingService( .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) .throttle(syncThrottle.inWholeMilliseconds) .collect { - /** - * Gets and performs the operations for active attachments which are - * pending download, upload, or delete. - */ - try { - val attachments = attachmentsService.getActiveAttachments() - // Performs pending operations and updates attachment states - handleSync(attachments) + attachmentsService.withLock { context -> + /** + * Gets and performs the operations for active attachments which are + * pending download, upload, or delete. + */ + try { + val attachments = context.getActiveAttachments() + // Performs pending operations and updates attachment states + handleSync(attachments, context) - // Cleanup archived attachments - deleteArchivedAttachments() - } catch (ex: Exception) { - if (ex is CancellationException) { - throw ex + // Cleanup archived attachments + deleteArchivedAttachments(context) + } catch (ex: Exception) { + if (ex is CancellationException) { + throw ex + } + // Rare exceptions caught here will be swallowed and retried on the + // next tick. + logger.e("Caught exception when processing attachments $ex") } - // Rare exceptions caught here will be swallowed and retried on the - // next tick. - logger.e("Caught exception when processing attachments $ex") } } } @@ -118,8 +121,8 @@ internal class SyncingService( syncJob.join() } - suspend fun deleteArchivedAttachments() = - attachmentsService.deleteArchivedAttachments { pendingDelete -> + suspend fun deleteArchivedAttachments(context: AttachmentContext) = + context.deleteArchivedAttachments { pendingDelete -> for (attachment in pendingDelete) { if (attachment.localUri == null) { continue @@ -134,7 +137,10 @@ internal class SyncingService( /** * Handle downloading, uploading or deleting of attachments */ - private suspend fun handleSync(attachments: List) { + private suspend fun handleSync( + attachments: List, + context: AttachmentContext, + ) { val updatedAttachments = mutableListOf() try { for (attachment in attachments) { @@ -157,7 +163,7 @@ internal class SyncingService( } // Update the state of processed attachments - attachmentsService.saveAttachments(updatedAttachments) + context.saveAttachments(updatedAttachments) } catch (error: Exception) { // We retry, on the next invocation, whenever there are errors on this level logger.e("Error during sync: ${error.message}") From c9b507735cc66d86caadf1e0db9384b831c24c25 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 7 Apr 2025 14:16:38 +0200 Subject: [PATCH 16/51] optional attachments in demo --- .../com/powersync/attachments/Attachment.kt | 26 +++ .../powersync/attachments/AttachmentQueue.kt | 13 +- .../powersync/attachments/AttachmentTable.kt | 8 +- .../com/powersync/attachments/README.md | 198 +++++++++++------- .../app/build.gradle.kts | 2 +- .../java/com/powersync/androidexample/App.kt | 31 +-- .../java/com/powersync/androidexample/Auth.kt | 9 +- .../androidexample/SupabaseRemoteStorage.kt | 5 +- .../androidexample/components/EditDialog.kt | 5 +- .../androidexample/powersync/Todo.kt | 8 +- 10 files changed, 199 insertions(+), 106 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt index 0810ceec..3693f267 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt @@ -21,18 +21,43 @@ public enum class AttachmentState { * Data class representing an attachment */ public data class Attachment( + /** + * Unique identifier + */ val id: String, + /** + * Timestamp for the last record update + */ val timestamp: Long = 0, + /** + * Attachment filename e.g. `[id].jpg` + */ val filename: String, + /** + * Current attachment state + */ val state: Int = AttachmentState.QUEUED_DOWNLOAD.ordinal, + /** + * Local URI pointing to the attachment file + */ val localUri: String? = null, + /** + * Attachment media type. Usually represented by a MIME type. + */ val mediaType: String? = null, + /** + * Attachment byte size + */ val size: Long? = null, /** * Specifies if the attachment has been synced locally before. This is particularly useful * for restoring archived attachments in edge cases. */ val hasSynced: Int = 0, + /** + * Extra attachment meta data. + */ + val metaData: String? = null, ) { public companion object { public fun fromCursor(cursor: SqlCursor): Attachment = @@ -45,6 +70,7 @@ public data class Attachment( size = cursor.getLongOptional("size"), state = cursor.getLong("state").toInt(), hasSynced = cursor.getLong("has_synced").toInt(), + metaData = cursor.getStringOptional("meta_data"), ) } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 37e4a8d8..bb080166 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -38,6 +38,10 @@ public data class WatchedAttachmentItem( * Filename to store the attachment with */ public val filename: String? = null, + /** + * Optional meta data for the attachment record + */ + public val metaData: String? = null, ) { init { require(fileExtension != null || filename != null) { @@ -298,6 +302,7 @@ public open class AttachmentQueue( id = item.id, filename = filename, state = AttachmentState.QUEUED_DOWNLOAD.ordinal, + metaData = item.metaData, ), ) } else if @@ -361,7 +366,8 @@ public open class AttachmentQueue( data: Flow, mediaType: String, fileExtension: String?, - updateHook: ((context: ConnectionContext, attachment: Attachment) -> Unit)? = null, + metaData: String?, + updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, ): Attachment = runWrappedSuspending { val id = db.get("SELECT uuid()") { it.getString(0)!! } @@ -385,12 +391,13 @@ public open class AttachmentQueue( mediaType = mediaType, state = AttachmentState.QUEUED_UPLOAD.ordinal, localUri = localUri, + metaData = metaData, ) /** * Allow consumers to set relationships to this attachment id */ - updateHook?.invoke(tx, attachment) + updateHook.invoke(tx, attachment) return@writeTransaction attachmentsService.upsertAttachment( attachment, @@ -409,7 +416,7 @@ public open class AttachmentQueue( @Throws(PowerSyncException::class, CancellationException::class) public open suspend fun deleteFile( attachmentId: String, - updateHook: ((context: ConnectionContext, attachment: Attachment) -> Unit)? = null, + updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, ): Attachment = runWrappedSuspending { val attachment = diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt index 978cf81d..deae177e 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt @@ -7,10 +7,7 @@ import com.powersync.db.schema.Table /** * Creates a PowerSync table for storing local attachment state */ -public fun createAttachmentsTable( - name: String, - additionalColumns: List? = null, -): Table = +public fun createAttachmentsTable(name: String): Table = Table( name = name, columns = @@ -22,6 +19,7 @@ public fun createAttachmentsTable( Column("media_type", ColumnType.TEXT), Column("state", ColumnType.INTEGER), Column("has_synced", ColumnType.INTEGER), - ).plus(additionalColumns ?: emptyList()), + Column("meta_data", ColumnType.TEXT), + ), localOnly = true, ) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index b14cceee..be6ae522 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -25,11 +25,6 @@ inspection workflow. The schema for the `checklist` table: ```kotlin -import com.powersync.attachments.AbstractAttachmentQueue -import com.powersync.attachments.createAttachmentsTable -import com.powersync.db.schema.Schema -import com.powersync.db.schema.Table - val checklists = Table( name = "checklists", columns = @@ -51,10 +46,9 @@ The `createAttachmentsTable` function defines the local only attachment state st An attachments table definition can be created with the following options. -| Option | Description | Default | -|---------------------|---------------------------------------------------------------------------------|-------------------------------| -| `name` | The name of the table | `attachments` | -| `additionalColumns` | An array of addition `Column` objects added to the default columns in the table | See below for default columns | +| Option | Description | Default | +|--------|-----------------------|---------------| +| `name` | The name of the table | `attachments` | The default columns in `AttachmentTable`: @@ -67,30 +61,23 @@ The default columns in `AttachmentTable`: | `timestamp` | `INTEGER` | The timestamp of last update to the attachment record | | `size` | `INTEGER` | The size of the attachment in bytes | | `has_synced` | `INTEGER` | Internal tracker which tracks if the attachment has ever been synced. This is used for caching/archiving purposes. | +| `meta_data` | `TEXT` | Any extra meta data for the attachment. JSON is usually a good choice. | ### Steps to implement -1. Create a new `AttachmentQueue` class that extends `AbstractAttachmentQueue` from - `com.powersync.attachments`. +1. Create an instance of `AttachmentQueue` from `com.powersync.attachments`. This class provides + default syncing utilities and implements a default sync strategy. This class is open and can be + overridden for custom functionality. ```kotlin -class AttachmentsQueue( - db: PowerSyncDatabase, - remoteStorage: RemoteStorageAdapter, - attachmentDirectoryName: String, -) : AbstractAttachmentQueue( - db, - remoteStorage, - attachmentDirectoryName = attachmentDirectoryName, - // See the class definition for more options -) { - - // An example implementation - override fun watchAttachments(): Flow> = - db.watch( - sql = - """ +val queue = AttachmentQueue( + db = db, + attachmentDirectory = attachmentDirectory, + remoteStorage = SupabaseRemoteStorage(supabase), + watchedAttachments = db.watch( + sql = + """ SELECT photo_id FROM @@ -98,30 +85,29 @@ class AttachmentsQueue( WHERE photo_id IS NOT NULL """, - ) { - WatchedAttachmentItem(id = it.getString("photo_id"), fileExtension = "jpg") - } -} + ) { + WatchedAttachmentItem(id = it.getString("photo_id"), fileExtension = "jpg") + } +) ``` -2. Implement `watchAttachments`, which returns a `Flow` of `WatchedAttachmentItem`. - The `WatchedAttachmentItem`s represent the attachments which should be present in the - application. We recommend using `PowerSync`'s `watch` query as shown above. In this example we - provide the `fileExtension` for all photos. This information could also be - obtained from the query if necessary. - -3. To instantiate an `AttachmentQueue`, one needs to provide an instance of - `PowerSyncDatabase` from PowerSync and an instance of `RemoteStorageAdapter`. The remote storage - is responsible for connecting to the attachments backend. See the `RemoteStorageAdapter` - interface - definition [here](https://github.com/powersync-ja/powersync-kotlin/blob/main/core/src/commonMain/kotlin/com.powersync/attachments/RemoteStorageAdapter.ts). - - -4. Implement a `RemoteStorageAdapter` which interfaces with a remote storage provider. This will be +* The `attachmentDirectory`, specifies where local attachment + files should be stored. This directory needs to be provided to the constructor. On Android + `"${applicationContext.filesDir.canonicalPath}/attachments"` is a good choice. +* The `remoteStorage` is responsible for connecting to the attachments backend. See the + `RemoteStorageAdapter` interface + definition [here](https://github.com/powersync-ja/powersync-kotlin/blob/main/core/src/commonMain/kotlin/com.powersync/attachments/RemoteStorageAdapter.ts). +* `watchAttachments` is a `Flow` of `WatchedAttachmentItem`. + The `WatchedAttachmentItem`s represent the attachments which should be present in the + application. We recommend using `PowerSync`'s `watch` query as shown above. In this example we + provide the `fileExtension` for all photos. This information could also be + obtained from the query if necessary. + +2. Implement a `RemoteStorageAdapter` which interfaces with a remote storage provider. This will be used for downloading, uploading and deleting attachments. ```kotlin -val remote = object : RemoteStorageAdapter() { +val remote = object : RemoteStorage() { override suspend fun uploadFile( fileData: Flow, attachment: Attachment, @@ -141,19 +127,13 @@ val remote = object : RemoteStorageAdapter() { ``` -5. Instantiate a new `AttachmentQueue` and call `startSync()` to start syncing attachments. +3. Call `startSync()` to start syncing attachments. ```kotlin - val queue = - AttachmentsQueue( - db = database, - remoteStorage = remote, - ) - queue.startSync() ``` -7. Finally, to create an attachment and add it to the queue, call `saveFile()`. This method will +4. Finally, to create an attachment and add it to the queue, call `saveFile()`. This method will save the file to the local storage, create an attachment record which queues the file for upload to the remote storage and allows assigning the newly created attachment ID to a checklist item. @@ -182,6 +162,46 @@ queue.saveFile( } ``` +#### Handling Errors + +The attachment queue automatically retries failed sync operations. Retries continue indefinitely +until success. A `SyncErrorHanlder` can be provided to the `AttachmentQueue` constructor. This +handler provides methods which are invoked on a remote sync exception. The handler can return a +Boolean which indicates if the attachment sync should be retried or archived. + +```kotlin +val errorHandler = object : SyncErrorHandler { + override suspend fun onDownloadError( + attachment: Attachment, + exception: Exception + ): Boolean { + TODO("Return if the attachment sync should be retried") + } + + override suspend fun onUploadError( + attachment: Attachment, + exception: Exception + ): Boolean { + TODO("Return if the attachment sync should be retried") + } + + override suspend fun onDeleteError( + attachment: Attachment, + exception: Exception + ): Boolean { + TODO("Return if the attachment sync should be retried") + } + +} + +// Pass the handler to the queue constructor +val queue = AttachmentQueue( +// ..., + errorHandler = errorHandler, +// ... +) +``` + # Implementation details ## Attachment State @@ -202,42 +222,80 @@ The state of an attachment can be one of the following: The `AttachmentQueue` sets a watched query on the `attachments` table, for record in the `QUEUED_UPLOAD`, `QUEUED_DELETE` and `QUEUED_DOWNLOAD` state. An event loop triggers calls to the -remote storage for -these operations. +remote storage for these operations. In addition to watching for changes, the `AttachmentQueue` also triggers a sync periodically. -This will retry any failed uploads/downloads, in particular after the app was offline. - -By default, this is every 30 seconds, but can be configured by setting `syncInterval` in the +This will retry any failed uploads/downloads, in particular after the app was offline. By default, +this is every 30 seconds, but can be configured by setting `syncInterval` in the `AttachmentQueue` constructor options, or disabled by setting the interval to `0`. +### Watching State + +The `watchedAttachments` flow provided to the `AttachmentQueue` constructor is used to reconcile the +local Attachment state. Each emission of the flow should represent the current attachment state. The +updated state is constantly compared to the current queue state. Items are queued based off the +difference. + +* A new watched item which is not present in the current queue is treated as an upstream Attachment + creation which needs to be downloaded. + * An attachment record is create using the provided watched item. The filename will be inferred + using a default filename resolver if it has not been provided in the watched item. + * The syncing service will attempt to download the attachment from the remote storage. + * The attachment will be saved to the local filesystem. The `local_uri` on the attachment record + will be updated. + * The attachment state will be updated to `SYNCED` +* Local attachments are archived if the watched state no longer includes the item. Archived items + are cached and can be restored if the watched state includes them in future. The number of cached + items is defined by the `archivedCacheLimit` parameter in the `AttachmentQueue` constructor. Items + are deleted once the cache limit is reached. + ### Uploading +The `saveFile` method provides a simple method for creating attachments which should be uploaded to +the backend. This method accepts the raw file content and meta data. This function: + +* Persists the attachment to the local filesystem +* Creates an attachment record linked to the local attachment file. +* Queues the attachment for upload. +* Allows assigning the attachment to relational data. + * It's important to assign the attachment to relational data since this data is constantly + watched and should always represent the attachment queue state. Failure to assign the + attachment could result in a failed upload. The attachment record will be archived. + +The sync process after calling `saveFile` is: + - An `AttachmentRecord` is created or updated with a state of `QUEUED_UPLOAD`. +- The `RemoteStorage` `uploadFile` function is called with the `Attachment` record. - The `AttachmentQueue` picks this up and upon successful upload to the remote storage, sets the - state to - `SYNCED`. + state to `SYNCED`. - If the upload is not successful, the record remains in `QUEUED_UPLOAD` state and uploading will be retried when syncing triggers again. Retries can be stopped by providing an `errorHandler`. ### Downloading +Attachments are schedules for download when the `watchedAttachments` flow emits a +`WatchedAttachmentItem` which is not present in the queue. + - An `AttachmentRecord` is created or updated with `QUEUED_DOWNLOAD` state. -- The watched query from `watchAttachments` adds the attachment `id` into a queue of IDs to download - and triggers the download process -- If the photo is not on the device, it is downloaded from cloud storage. -- Writes file to the user's local storage. +- The `RemoteStorage` `downloadFile` function is called with the attachment record. +- The received data is persisted to the local filesystem. - If this is successful, update the `AttachmentRecord` state to `SYNCED`. - If any of these fail, the download is retried in the next sync trigger. ### Deleting attachments -When an attachment is deleted by a user action or cache expiration: +Local attachments are archived and deleted (locally) if the `watchedAttachments` flow no longer +references them. Archived attachments are deleted locally after cache invalidation. + +In some cases users might want to explicitly delete an attachment in the backend. The `deleteFile` +function provides a mechanism for this. This function: -- An `AttachmentRecord` is created or updated with `QUEUED_DELETE` state. -- The `RemoteStorage`'s `deleteFile` method will be called. -- Local file (if exists) is deleted. -- If successful, the `AttachmentRecord` is deleted. +* Deletes the attachment on the local filesystem +* Updates the record to the `QUEUED_DELETE` state +* Allows removing assignments to relational data. + * It's important to unassign the attachment from relational data since this data is constantly + watched and should always represent the attachment queue state. Failure to unassign the + attachment could result in a failed delete. ### Expire Cache diff --git a/demos/android-supabase-todolist/app/build.gradle.kts b/demos/android-supabase-todolist/app/build.gradle.kts index 24c390b4..6157eeb3 100644 --- a/demos/android-supabase-todolist/app/build.gradle.kts +++ b/demos/android-supabase-todolist/app/build.gradle.kts @@ -47,7 +47,7 @@ android { buildConfigField( "String", "SUPABASE_ATTACHMENT_BUCKET", - "\"${getLocalProperty("SUPABASE_ATTACHMENT_BUCKET", "")}\"", + "\"${getLocalProperty("SUPABASE_ATTACHMENT_BUCKET", "null")}\"", ) buildConfigField("String", "POWERSYNC_URL", "\"${getLocalProperty("POWERSYNC_URL", "")}\"") } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index b376e8ab..65115cb8 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -46,17 +46,20 @@ fun App(cameraService: CameraService, attachmentDirectory: String) { val db = remember { PowerSyncDatabase(driverFactory, schema, dbFilename = "333.sqlite") } val attachments = - remember { AttachmentQueue( - db = db, remoteStorage = SupabaseRemoteStorage(supabase), - attachmentDirectory = attachmentDirectory, - watchedAttachments = db.watch( - "SELECT photo_id from todos WHERE photo_id IS NOT NULL" - ) { - WatchedAttachmentItem( - id = it.getString("photo_id"), - fileExtension = "jpg" - ) - }) } + remember { + if (BuildConfig.SUPABASE_ATTACHMENT_BUCKET != "null") { + AttachmentQueue( + db = db, remoteStorage = SupabaseRemoteStorage(supabase), + attachmentDirectory = attachmentDirectory, + watchedAttachments = db.watch( + "SELECT photo_id from todos WHERE photo_id IS NOT NULL" + ) { + WatchedAttachmentItem( + id = it.getString("photo_id"), + fileExtension = "jpg" + ) + }) + } else {null} } val syncStatus = db.currentStatus val status by syncStatus.asFlow().collectAsState(syncStatus) @@ -67,7 +70,7 @@ fun App(cameraService: CameraService, attachmentDirectory: String) { val authViewModel = remember { - AuthViewModel(supabase, db, attachments, navController) + AuthViewModel(supabase, db, navController, attachments) } val authState by authViewModel.authState.collectAsState() @@ -146,8 +149,8 @@ fun App(cameraService: CameraService, attachmentDirectory: String) { onTextChanged = todos.value::onEditorTextChanged, onDoneChanged = todos.value::onEditorDoneChanged, onPhotoClear = todos.value::onPhotoDelete, - onPhotoCapture = {todos.value::onPhotoCapture.invoke(cameraService)} - + onPhotoCapture = {todos.value::onPhotoCapture.invoke(cameraService)}, + attachmentsSupported = attachments != null ) } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt index 7f06496b..1b62d9c4 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt @@ -6,7 +6,6 @@ import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase import com.powersync.attachments.AttachmentQueue import com.powersync.connector.supabase.SupabaseConnector -import com.powersync.demos.screens.SignInScreen import io.github.jan.supabase.auth.status.RefreshFailureCause import io.github.jan.supabase.auth.status.SessionStatus import kotlinx.coroutines.flow.MutableStateFlow @@ -22,8 +21,8 @@ sealed class AuthState { internal class AuthViewModel( private val supabase: SupabaseConnector, private val db: PowerSyncDatabase, - private val attachmentsQueue: AttachmentQueue, private val navController: NavController, + private val attachmentsQueue: AttachmentQueue? ) : ViewModel() { private val _authState = MutableStateFlow(AuthState.SignedOut) val authState: StateFlow = _authState @@ -38,7 +37,7 @@ internal class AuthViewModel( _authState.value = AuthState.SignedIn _userId.value = it.session.user?.id db.connect(supabase) - attachmentsQueue.startSync() + attachmentsQueue?.startSync() if (navController.currentScreen.value is Screen.SignIn || navController.currentScreen.value is Screen.SignUp) { navController.navigate(Screen.Home) @@ -79,8 +78,8 @@ internal class AuthViewModel( suspend fun signOut() { try { - attachmentsQueue.clearQueue() - attachmentsQueue.close() + attachmentsQueue?.clearQueue() + attachmentsQueue?.close() supabase.signOut() } catch (e: Exception) { Logger.e("Error signing out: $e") diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt index 743a9676..925e2302 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt @@ -1,15 +1,14 @@ package com.powersync.androidexample import com.powersync.attachments.Attachment -import com.powersync.attachments.RemoteStorageAdapter +import com.powersync.attachments.RemoteStorage import com.powersync.connector.supabase.SupabaseConnector import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.flowOf class SupabaseRemoteStorage( val connector: SupabaseConnector, -) : RemoteStorageAdapter { +) : RemoteStorage { override suspend fun uploadFile( fileData: Flow, attachment: Attachment, diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt index c4b56ba0..7a51ac37 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt @@ -38,7 +38,8 @@ internal fun EditDialog( onTextChanged: (String) -> Unit, onDoneChanged: (Boolean) -> Unit, onPhotoClear: () -> Unit, - onPhotoCapture: () -> Unit + onPhotoCapture: () -> Unit, + attachmentsSupported: Boolean = false ) { EditDialog( onCloseRequest = onCloseClicked, @@ -65,6 +66,7 @@ internal fun EditDialog( item.photoURI?.let { BitmapFactory.decodeFile(it)?.asImageBitmap() } } + if (attachmentsSupported == true) { Box( modifier = Modifier .clickable { if (item.photoId == null) onPhotoCapture() } @@ -90,6 +92,7 @@ internal fun EditDialog( } } } + } } } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt index 139c6193..ebb2e557 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt @@ -20,7 +20,7 @@ import kotlinx.datetime.Clock internal class Todo( private val db: PowerSyncDatabase, - private val attachmentsQueue: AttachmentQueue, + private val attachmentsQueue: AttachmentQueue?, private val userId: String?, ) : ViewModel() { private val _inputText = MutableStateFlow("") @@ -77,7 +77,7 @@ internal class Todo( fun onItemDeleteClicked(item: TodoItem) { viewModelScope.launch { if (item.photoId != null) { - attachmentsQueue.deleteFile(item.photoId) + attachmentsQueue?.deleteFile(item.photoId) } db.writeTransaction { tx -> tx.execute("DELETE FROM $TODOS_TABLE WHERE id = ?", listOf(item.id)) @@ -144,7 +144,7 @@ internal class Todo( return@launch } } - val attachment = attachmentsQueue.saveFile(data = flowOf(photoData), mediaType = "image/jped", fileExtension = "jpg" ) {tx, attachment -> + val attachment = attachmentsQueue!!.saveFile(data = flowOf(photoData), mediaType = "image/jped", fileExtension = "jpg" ) {tx, attachment -> tx.execute("UPDATE $TODOS_TABLE SET photo_id = ? WHERE id = ?", listOf(attachment.id, item.id)) } @@ -155,7 +155,7 @@ internal class Todo( fun onPhotoDelete() { viewModelScope.launch { val item = requireNotNull(_editingItem.value) - attachmentsQueue.deleteFile(item.photoId!!) {tx, _ -> + attachmentsQueue!!.deleteFile(item.photoId!!) {tx, _ -> tx.execute("UPDATE $TODOS_TABLE SET photo_id = NULL WHERE id = ?", listOf(item.id)) } updateEditingItem(item = item) {it.copy(photoURI = null)} From 09eb6168c5d804b77db2361f4a51df41bf850d75 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 7 Apr 2025 14:21:03 +0200 Subject: [PATCH 17/51] cleanup service --- .../powersync/attachments/AttachmentQueue.kt | 70 ++++++++++--------- .../attachments/AttachmentService.kt | 2 +- .../implementation/AttachmentServiceImpl.kt | 28 -------- 3 files changed, 39 insertions(+), 61 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index bb080166..4cff20ed 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -365,8 +365,8 @@ public open class AttachmentQueue( public open suspend fun saveFile( data: Flow, mediaType: String, - fileExtension: String?, - metaData: String?, + fileExtension: String? = null, + metaData: String? = null, updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, ): Attachment = runWrappedSuspending { @@ -382,27 +382,29 @@ public open class AttachmentQueue( * Starts a write transaction. The attachment record and relevant local relationship * assignment should happen in the same transaction. */ - db.writeTransaction { tx -> - val attachment = - Attachment( - id = id, - filename = filename, - size = fileSize, - mediaType = mediaType, - state = AttachmentState.QUEUED_UPLOAD.ordinal, - localUri = localUri, - metaData = metaData, - ) + attachmentsService.withLock { attachmentContext -> + db.writeTransaction { tx -> + val attachment = + Attachment( + id = id, + filename = filename, + size = fileSize, + mediaType = mediaType, + state = AttachmentState.QUEUED_UPLOAD.ordinal, + localUri = localUri, + metaData = metaData, + ) - /** - * Allow consumers to set relationships to this attachment id - */ - updateHook.invoke(tx, attachment) + /** + * Allow consumers to set relationships to this attachment id + */ + updateHook.invoke(tx, attachment) - return@writeTransaction attachmentsService.upsertAttachment( - attachment, - tx, - ) + return@writeTransaction attachmentContext.upsertAttachment( + attachment, + tx, + ) + } } } @@ -419,16 +421,18 @@ public open class AttachmentQueue( updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, ): Attachment = runWrappedSuspending { - val attachment = - attachmentsService.getAttachment(attachmentId) - ?: throw Exception("Attachment record with id $attachmentId was not found.") - - db.writeTransaction { tx -> - updateHook?.invoke(tx, attachment) - return@writeTransaction attachmentsService.upsertAttachment( - attachment.copy(state = AttachmentState.QUEUED_DELETE.ordinal), - tx, - ) + attachmentsService.withLock { attachmentContext -> + val attachment = + attachmentContext.getAttachment(attachmentId) + ?: throw Exception("Attachment record with id $attachmentId was not found.") + + db.writeTransaction { tx -> + updateHook?.invoke(tx, attachment) + return@writeTransaction attachmentContext.upsertAttachment( + attachment.copy(state = AttachmentState.QUEUED_DELETE.ordinal), + tx, + ) + } } } @@ -454,7 +458,9 @@ public open class AttachmentQueue( * Clears the attachment queue and deletes all attachment files */ public suspend fun clearQueue() { - attachmentsService.clearQueue() + attachmentsService.withLock { + it.clearQueue() + } // Remove the attachments directory localStorage.rmDir(attachmentDirectory) } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt index fe09bd0b..13fe91c3 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt @@ -73,7 +73,7 @@ public interface AttachmentContext { /** * Service for interacting with the local attachment records. */ -public interface AttachmentService : AttachmentContext { +public interface AttachmentService { /** * Watcher for changes to attachments table. * Once a change is detected it will initiate a sync of the attachments diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt index 79efab6f..03a7e584 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt @@ -2,11 +2,9 @@ package com.powersync.attachments.implementation import co.touchlab.kermit.Logger import com.powersync.PowerSyncDatabase -import com.powersync.attachments.Attachment import com.powersync.attachments.AttachmentContext import com.powersync.attachments.AttachmentService import com.powersync.attachments.AttachmentState -import com.powersync.db.internal.ConnectionContext import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.sync.Mutex @@ -68,30 +66,4 @@ public open class AttachmentServiceImpl( // We only use changes here to trigger a sync consolidation .map { Unit } } - - override fun upsertAttachment( - attachment: Attachment, - context: ConnectionContext, - ): Attachment = this.context.upsertAttachment(attachment = attachment, context = context) - - override suspend fun deleteAttachment(id: String): Unit = withLock { it.deleteAttachment(id) } - - override suspend fun ignoreAttachment(id: String): Unit = withLock { it.ignoreAttachment(id) } - - override suspend fun getAttachment(id: String): Attachment? = withLock { it.getAttachment(id) } - - override suspend fun saveAttachment(attachment: Attachment): Attachment = withLock { it.saveAttachment(attachment) } - - override suspend fun saveAttachments(attachments: List): Unit = withLock { it.saveAttachments(attachments) } - - override suspend fun getAttachmentIds(): List = withLock { it.getAttachmentIds() } - - override suspend fun getAttachments(): List = withLock { it.getAttachments() } - - override suspend fun getActiveAttachments(): List = withLock { it.getActiveAttachments() } - - override suspend fun clearQueue(): Unit = withLock { it.clearQueue() } - - override suspend fun deleteArchivedAttachments(callback: suspend (List) -> Unit): Boolean = - withLock { it.deleteArchivedAttachments(callback) } } From 8d8196c67fd267cd6359cff761f978b2d8f46a24 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 7 Apr 2025 15:36:00 +0200 Subject: [PATCH 18/51] upate tests --- .../kotlin/com/powersync/AttachmentsTest.kt | 245 ++++++++---------- .../kotlin/com/powersync/DatabaseTest.kt | 1 - .../com/powersync/testutils/TestUtils.kt | 2 + .../powersync/attachments/AttachmentQueue.kt | 36 ++- .../com/powersync/attachments/README.md | 4 +- .../storage/IOLocalStorageAdapter.kt | 11 +- .../attachments/sync/SyncingService.kt | 130 +++++----- 7 files changed, 229 insertions(+), 200 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 19a15db9..04420b8d 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -11,8 +11,10 @@ import com.powersync.attachments.WatchedAttachmentItem import com.powersync.attachments.createAttachmentsTable import com.powersync.db.getString import com.powersync.db.schema.Schema +import com.powersync.db.schema.Table import com.powersync.testutils.MockedRemoteStorage import com.powersync.testutils.UserRow +import com.powersync.testutils.databaseTest import com.powersync.testutils.getTempDir import dev.mokkery.answering.throws import dev.mokkery.everySuspend @@ -22,51 +24,15 @@ import dev.mokkery.matcher.matching import dev.mokkery.mock import dev.mokkery.spy import dev.mokkery.verifySuspend +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe import kotlinx.coroutines.flow.flowOf -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.test.runTest -import kotlin.test.AfterTest -import kotlin.test.BeforeTest +import kotlinx.io.files.Path import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertNotNull -import kotlin.test.assertNull -import kotlin.test.assertTrue -import kotlin.time.Duration.Companion.minutes @OptIn(ExperimentalKermitApi::class) class AttachmentsTest { - private lateinit var database: PowerSyncDatabase - - private fun openDB() = - PowerSyncDatabase( - factory = com.powersync.testutils.factory, - schema = Schema(UserRow.table, createAttachmentsTable("attachments")), - dbFilename = "testdb", - ) - - @BeforeTest - fun setupDatabase() { - database = openDB() - - runBlocking { - database.disconnectAndClear(true) - } - } - - @AfterTest - fun tearDown() { - runBlocking { - if (!database.closed) { - database.disconnectAndClear(true) - database.close() - } - } - com.powersync.testutils - .cleanup("testdb") - } - - fun watchAttachments() = + fun watchAttachments(database: PowerSyncDatabase) = database.watch( sql = """ @@ -84,18 +50,40 @@ class AttachmentsTest { ) } + suspend fun updateSchema(db: PowerSyncDatabase) { + db.updateSchema( + Schema( + tables = + listOf( + UserRow.table, + createAttachmentsTable("attachments"), + ), + ), + ) + } + + fun getAttachmentsDir() = Path(getTempDir(), "attachments").toString() + @Test fun testAttachmentDownload() = - runTest(timeout = 5.minutes) { - turbineScope(timeout = 5.minutes) { + databaseTest { + turbineScope { + updateSchema(database) + val remote = spy(MockedRemoteStorage()) + // Monitor the attachments table for testing + val attachmentQuery = + database + .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } + .testIn(this) + val queue = AttachmentQueue( db = database, remoteStorage = remote, - attachmentDirectory = getTempDir(), - watchedAttachments = watchAttachments(), + attachmentsDirectory = getAttachmentsDir(), + watchedAttachments = watchAttachments(database), /** * Sets the cache limit to zero for this test. Archived records will * immediately be deleted. @@ -103,18 +91,19 @@ class AttachmentsTest { archivedCacheLimit = 0, ) - queue.startSync() + doOnCleanup { + queue.stopSyncing() + attachmentQuery.cancel() + queue.clearQueue() + queue.close() + } - // Monitor the attachments table for testing - val attachmentQuery = - database - .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } - .testIn(this) + queue.startSync() val result = attachmentQuery.awaitItem() // There should not be any attachment records here - assertEquals(expected = 0, actual = result.size) + result.size shouldBe 0 // Create a user with a photo_id specified. // This code did not save an attachment before assigning a photo_id. @@ -129,13 +118,7 @@ class AttachmentsTest { ) var attachmentRecord = attachmentQuery.awaitItem().first() - assertNotNull( - attachmentRecord, - """ - An attachment record should be created after creating a user with a photo_id - " - """.trimIndent(), - ) + attachmentRecord shouldNotBe null /** * The timing of the watched query resolving might differ slightly. @@ -149,14 +132,14 @@ class AttachmentsTest { attachmentRecord = attachmentQuery.awaitItem().first() } - assertEquals(expected = AttachmentState.SYNCED.ordinal, attachmentRecord.state) + attachmentRecord.state shouldBe AttachmentState.SYNCED.ordinal // A download should have been attempted for this file verifySuspend { remote.downloadFile(attachmentMatcher(attachmentRecord)) } // A file should now exist val localUri = attachmentRecord.localUri!! - assertTrue { queue.localStorage.fileExists(localUri) } + queue.localStorage.fileExists(localUri) shouldBe true // Now clear the user's photo_id. The attachment should be archived database.execute( @@ -180,28 +163,35 @@ class AttachmentsTest { } // The record should have been deleted - assertNull(nextRecord) + nextRecord shouldBe null // The file should have been deleted from storage - assertEquals(expected = false, actual = queue.localStorage.fileExists(localUri)) + val exists = queue.localStorage.fileExists(localUri) + exists shouldBe false attachmentQuery.cancel() - queue.close() } } @Test fun testAttachmentUpload() = - runTest { + databaseTest { turbineScope { + updateSchema(database) val remote = spy(MockedRemoteStorage()) + // Monitor the attachments table for testing + val attachmentQuery = + database + .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } + .testIn(this) + val queue = AttachmentQueue( db = database, remoteStorage = remote, - attachmentDirectory = getTempDir(), - watchedAttachments = watchAttachments(), + attachmentsDirectory = getAttachmentsDir(), + watchedAttachments = watchAttachments(database), /** * Sets the cache limit to zero for this test. Archived records will * immediately be deleted. @@ -209,18 +199,19 @@ class AttachmentsTest { archivedCacheLimit = 0, ) - queue.startSync() + doOnCleanup { + queue.stopSyncing() + queue.clearQueue() + queue.close() + attachmentQuery.cancel() + } - // Monitor the attachments table for testing - val attachmentQuery = - database - .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } - .testIn(this) + queue.startSync() val result = attachmentQuery.awaitItem() // There should not be any attachment records here - assertEquals(expected = 0, actual = result.size) + result.size shouldBe 0 /** * Creates an attachment given a flow of bytes (the file data) then assigns this to @@ -244,17 +235,14 @@ class AttachmentsTest { } var attachmentRecord = attachmentQuery.awaitItem().first() - assertNotNull(attachmentRecord) + attachmentRecord shouldNotBe null if (attachmentRecord.state == AttachmentState.QUEUED_UPLOAD.ordinal) { // Wait for it to be synced attachmentRecord = attachmentQuery.awaitItem().first() } - assertEquals( - expected = AttachmentState.SYNCED.ordinal, - attachmentRecord.state, - ) + attachmentRecord.state shouldBe AttachmentState.SYNCED.ordinal // A download should have been attempted for this file verifySuspend { @@ -266,7 +254,7 @@ class AttachmentsTest { // A file should now exist val localUri = attachmentRecord.localUri!! - assertTrue { queue.localStorage.fileExists(localUri) } + queue.localStorage.fileExists(localUri) shouldBe true // Now clear the user's photo_id. The attachment should be archived database.execute( @@ -284,46 +272,54 @@ class AttachmentsTest { } // The record should have been deleted - assertNull(nextRecord) + nextRecord shouldBe null // The file should have been deleted from storage - assertEquals(expected = false, actual = queue.localStorage.fileExists(localUri)) + queue.localStorage.fileExists(localUri) shouldBe false attachmentQuery.cancel() - queue.close() } } @Test fun testAttachmentCachedDownload() = - runTest { + databaseTest { turbineScope { + updateSchema(database) + val remote = spy(MockedRemoteStorage()) + // Monitor the attachments table for testing + val attachmentQuery = + database + .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } + .testIn(this) + val queue = AttachmentQueue( db = database, remoteStorage = remote, - attachmentDirectory = getTempDir(), - watchedAttachments = watchAttachments(), + attachmentsDirectory = getAttachmentsDir(), + watchedAttachments = watchAttachments(database), /** * Keep some items in the cache */ archivedCacheLimit = 10, ) - queue.startSync() + doOnCleanup { + queue.stopSyncing() + queue.clearQueue() + queue.close() + attachmentQuery.cancel() + } - // Monitor the attachments table for testing - val attachmentQuery = - database - .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } - .testIn(this) + queue.startSync() val result = attachmentQuery.awaitItem() // There should not be any attachment records here - assertEquals(expected = 0, actual = result.size) + result.size shouldBe 0 // Create a user with a photo_id specified. // This code did not save an attachment before assigning a photo_id. @@ -338,13 +334,7 @@ class AttachmentsTest { ) var attachmentRecord = attachmentQuery.awaitItem().first() - assertNotNull( - attachmentRecord, - """ - An attachment record should be created after creating a user with a photo_id - " - """.trimIndent(), - ) + attachmentRecord shouldNotBe null /** * The timing of the watched query resolving might differ slightly. @@ -358,14 +348,14 @@ class AttachmentsTest { attachmentRecord = attachmentQuery.awaitItem().first() } - assertEquals(expected = AttachmentState.SYNCED.ordinal, attachmentRecord.state) + attachmentRecord.state shouldBe AttachmentState.SYNCED.ordinal // A download should have been attempted for this file verifySuspend { remote.downloadFile(attachmentMatcher(attachmentRecord)) } // A file should now exist val localUri = attachmentRecord.localUri!! - assertTrue { queue.localStorage.fileExists(localUri) } + queue.localStorage.fileExists(localUri) shouldBe true // Now clear the user's photo_id. The attachment should be archived database.execute( @@ -378,10 +368,7 @@ class AttachmentsTest { ) attachmentRecord = attachmentQuery.awaitItem().first() - assertEquals( - expected = AttachmentState.ARCHIVED.ordinal, - actual = attachmentRecord.state, - ) + attachmentRecord.state shouldBe AttachmentState.ARCHIVED.ordinal // Now if we set the photo_id, the archived record should be restored database.execute( @@ -395,31 +382,35 @@ class AttachmentsTest { ) attachmentRecord = attachmentQuery.awaitItem().first() - assertEquals( - expected = AttachmentState.SYNCED.ordinal, - actual = attachmentRecord.state, - ) + attachmentRecord.state shouldBe AttachmentState.SYNCED.ordinal attachmentQuery.cancel() - queue.close() } } @Test fun testSkipFailedDownload() = - runTest { + databaseTest { turbineScope { + updateSchema(database) + val remote = mock { everySuspend { downloadFile(any()) } throws (Exception("Test error")) } + // Monitor the attachments table for testing + val attachmentQuery = + database + .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } + .testIn(this) + val queue = AttachmentQueue( db = database, remoteStorage = remote, - attachmentDirectory = getTempDir(), - watchedAttachments = watchAttachments(), + attachmentsDirectory = getAttachmentsDir(), + watchedAttachments = watchAttachments(database), archivedCacheLimit = 0, errorHandler = object : SyncErrorHandler { @@ -439,19 +430,19 @@ class AttachmentsTest { ): Boolean = false }, ) + doOnCleanup { + queue.stopSyncing() + queue.clearQueue() + queue.close() + attachmentQuery.cancel() + } queue.startSync() - // Monitor the attachments table for testing - val attachmentQuery = - database - .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } - .testIn(this) - val result = attachmentQuery.awaitItem() // There should not be any attachment records here - assertEquals(expected = 0, actual = result.size) + result.size shouldBe 0 // Create a user with a photo_id specified. // This code did not save an attachment before assigning a photo_id. @@ -466,22 +457,16 @@ class AttachmentsTest { ) var attachmentRecord = attachmentQuery.awaitItem().first() - assertNotNull(attachmentRecord) + attachmentRecord shouldNotBe null - assertEquals( - expected = AttachmentState.QUEUED_DOWNLOAD.ordinal, - actual = attachmentRecord.state, - ) + attachmentRecord.state shouldBe AttachmentState.QUEUED_DOWNLOAD.ordinal // The download should fail. We don't specify a retry. The record should be archived. attachmentRecord = attachmentQuery.awaitItem().first() - assertEquals( - expected = AttachmentState.ARCHIVED.ordinal, - actual = attachmentRecord.state, - ) + + attachmentRecord.state shouldBe AttachmentState.ARCHIVED.ordinal attachmentQuery.cancel() - queue.close() } } } diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt index c82af2cf..d65d6eb4 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt @@ -7,7 +7,6 @@ import com.powersync.db.schema.Schema import com.powersync.testutils.UserRow import com.powersync.testutils.databaseTest import com.powersync.testutils.getTempDir -import com.powersync.testutils.isIOS import com.powersync.testutils.waitFor import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.collections.shouldHaveSize diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestUtils.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestUtils.kt index 7dcd69bd..fb1c92d6 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestUtils.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/testutils/TestUtils.kt @@ -133,6 +133,8 @@ internal class ActiveDatabaseTest( } suspend fun cleanup() { + // Execute in reverse order + cleanupItems.reverse() for (item in cleanupItems) { item() } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 4cff20ed..f403ab70 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -69,7 +69,7 @@ public open class AttachmentQueue( * Directory name where attachment files will be written to disk. * This will be created if it does not exist */ - private val attachmentDirectory: String, + private val attachmentsDirectory: String, /** * A flow for the current state of local attachments * ```kotlin @@ -186,15 +186,15 @@ public open class AttachmentQueue( throw Exception("Attachment queue has been closed") } // Ensure the directory where attachments are downloaded, exists - localStorage.makeDir(attachmentDirectory) + localStorage.makeDir(attachmentsDirectory) subdirectories?.forEach { subdirectory -> - localStorage.makeDir(Path(attachmentDirectory, subdirectory).toString()) + localStorage.makeDir(Path(attachmentsDirectory, subdirectory).toString()) } val scope = CoroutineScope(Dispatchers.IO) - syncingService.startPeriodicSync(syncInterval) + syncingService.startSync(syncInterval) // Listen for connectivity changes syncStatusJob = @@ -224,6 +224,29 @@ public open class AttachmentQueue( } } + /** + * Stops syncing. Syncing may be resumed with [startSync]. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun stopSyncing(): Unit = + runWrappedSuspending { + mutex.withLock { + if (closed) { + return@runWrappedSuspending + } + + syncStatusJob?.cancel() + syncStatusJob?.join() + + syncingService.stopSync() + } + } + + /** + * Closes the queue. + * The queue cannot be used after closing. + * A new queue should be created. + */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun close(): Unit = runWrappedSuspending { @@ -235,7 +258,6 @@ public open class AttachmentQueue( syncStatusJob?.cancel() syncStatusJob?.join() syncingService.close() - closed = true } } @@ -440,7 +462,7 @@ public open class AttachmentQueue( * Return users storage directory with the attachmentPath use to load the file. * Example: filePath: "attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg" */ - public open fun getLocalUri(filename: String): String = Path(attachmentDirectory, filename).toString() + public open fun getLocalUri(filename: String): String = Path(attachmentsDirectory, filename).toString() /** * Removes all archived items @@ -462,6 +484,6 @@ public open class AttachmentQueue( it.clearQueue() } // Remove the attachments directory - localStorage.rmDir(attachmentDirectory) + localStorage.rmDir(attachmentsDirectory) } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index be6ae522..d7c19e73 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -73,7 +73,7 @@ The default columns in `AttachmentTable`: val queue = AttachmentQueue( db = db, - attachmentDirectory = attachmentDirectory, + attachmentsDirectory = attachmentsDirectory, remoteStorage = SupabaseRemoteStorage(supabase), watchedAttachments = db.watch( sql = @@ -91,7 +91,7 @@ val queue = AttachmentQueue( ) ``` -* The `attachmentDirectory`, specifies where local attachment +* The `attachmentsDirectory`, specifies where local attachment files should be stored. This directory needs to be provided to the constructor. On Android `"${applicationContext.filesDir.canonicalPath}/attachments"` is a good choice. * The `remoteStorage` is responsible for connecting to the attachments backend. See the diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt index a50035e1..c5d96c49 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt @@ -85,7 +85,16 @@ public class IOLocalStorageAdapter : LocalStorage { public override suspend fun rmDir(path: String): Unit = runWrappedSuspending { withContext(Dispatchers.IO) { - SystemFileSystem.delete(Path(path)) + for (item in SystemFileSystem.list(Path(path))) { + // Can't delete directories with files in them. Need to go down the file tree + // and clear the directory. + val meta = SystemFileSystem.metadataOrNull(item) + if (meta?.isDirectory == true) { + rmDir(item.toString()) + } else if (meta?.isRegularFile == true) { + SystemFileSystem.delete(item) + } + } } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index d92f7c3d..46cb920b 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -30,7 +30,7 @@ import kotlin.time.Duration.Companion.seconds /** * Service used to sync attachments between local and remote storage */ -internal class SyncingService( +public class SyncingService( private val remoteStorage: RemoteStorage, private val localStorage: LocalStorage, private val attachmentsService: AttachmentService, @@ -41,87 +41,99 @@ internal class SyncingService( ) { private val scope = CoroutineScope(Dispatchers.IO) private val mutex = Mutex() - private val syncJob: Job - private var periodicSyncTrigger: Job? = null + private var syncJob: Job? = null /** * Used to trigger the sync process either manually or periodically */ private val syncTriggerFlow = MutableSharedFlow(replay = 0) - init { - syncJob = - scope.launch { - merge( - // Handles manual triggers for sync events - syncTriggerFlow.asSharedFlow(), - // Triggers the sync process whenever an underlaying change to the - // attachments table happens - attachmentsService - .watchActiveAttachments(), - ) - // We only use these flows to trigger the process. We can skip multiple invocations - // while we are processing. We will always process on the trailing edge. - // This buffer operation should automatically be applied to all merged sources. - .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) - .throttle(syncThrottle.inWholeMilliseconds) - .collect { - attachmentsService.withLock { context -> - /** - * Gets and performs the operations for active attachments which are - * pending download, upload, or delete. - */ - try { - val attachments = context.getActiveAttachments() - // Performs pending operations and updates attachment states - handleSync(attachments, context) - - // Cleanup archived attachments - deleteArchivedAttachments(context) - } catch (ex: Exception) { - if (ex is CancellationException) { - throw ex - } - // Rare exceptions caught here will be swallowed and retried on the - // next tick. - logger.e("Caught exception when processing attachments $ex") - } - } - } - } - } - /** - * Periodically sync attachments and delete archived attachments + * Starts syncing operations */ - suspend fun startPeriodicSync(period: Duration): Unit = + public suspend fun startSync(period: Duration = 30.seconds): Unit = mutex.withLock { - periodicSyncTrigger?.cancel() + syncJob?.cancel() + syncJob?.join() - periodicSyncTrigger = + syncJob = scope.launch { - logger.i("Periodically syncing attachments") - syncTriggerFlow.emit(Unit) - delay(period) + val watchJob = + launch { + merge( + // Handles manual triggers for sync events + syncTriggerFlow.asSharedFlow(), + // Triggers the sync process whenever an underlaying change to the + // attachments table happens + attachmentsService + .watchActiveAttachments(), + ) + // We only use these flows to trigger the process. We can skip multiple invocations + // while we are processing. We will always process on the trailing edge. + // This buffer operation should automatically be applied to all merged sources. + .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) + .throttle(syncThrottle.inWholeMilliseconds) + .collect { + attachmentsService.withLock { context -> + /** + * Gets and performs the operations for active attachments which are + * pending download, upload, or delete. + */ + try { + val attachments = context.getActiveAttachments() + // Performs pending operations and updates attachment states + handleSync(attachments, context) + + // Cleanup archived attachments + deleteArchivedAttachments(context) + } catch (ex: Exception) { + if (ex is CancellationException) { + throw ex + } + // Rare exceptions caught here will be swallowed and retried on the + // next tick. + logger.e("Caught exception when processing attachments $ex") + } + } + } + } + + val periodicJob = + launch { + logger.i("Periodically syncing attachments") + syncTriggerFlow.emit(Unit) + delay(period) + } + + watchJob.join() + periodicJob.join() } } /** * Enqueues a sync operation */ - suspend fun triggerSync() { + public suspend fun triggerSync() { syncTriggerFlow.emit(Unit) } - suspend fun close(): Unit = + /** + * Stops syncing operations + */ + public suspend fun stopSync(): Unit = mutex.withLock { - periodicSyncTrigger?.cancel() - periodicSyncTrigger?.join() - syncJob.cancel() - syncJob.join() + syncJob?.cancel() + syncJob?.join() } - suspend fun deleteArchivedAttachments(context: AttachmentContext) = + /** + * Closes the syncing service. + */ + public suspend fun close() { + stopSync() + } + + public suspend fun deleteArchivedAttachments(context: AttachmentContext): Boolean = context.deleteArchivedAttachments { pendingDelete -> for (attachment in pendingDelete) { if (attachment.localUri == null) { From 4efc67d06e5267550870ce831b9e6cc626e61704 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 7 Apr 2025 15:59:38 +0200 Subject: [PATCH 19/51] fix tests --- .../kotlin/com/powersync/DatabaseTest.kt | 14 +++++++++++--- .../main/java/com/powersync/androidexample/App.kt | 2 +- .../com/powersync/androidexample/powersync/Todo.kt | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt index d65d6eb4..63f1478d 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt @@ -7,6 +7,7 @@ import com.powersync.db.schema.Schema import com.powersync.testutils.UserRow import com.powersync.testutils.databaseTest import com.powersync.testutils.getTempDir +import com.powersync.testutils.isIOS import com.powersync.testutils.waitFor import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.collections.shouldHaveSize @@ -231,9 +232,16 @@ class DatabaseTest { fun openDBWithDirectory() = databaseTest { val tempDir = - getTempDir() - ?: // SQLiteR, which is used on iOS, does not support opening dbs from directories - return@databaseTest + if (isIOS()) { + null + } else { + getTempDir() + } + + if (tempDir == null) { + // SQLiteR, which is used on iOS, does not support opening dbs from directories + return@databaseTest + } // On platforms that support it, openDatabase() from our test utils should use a temporary // location. diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index 65115cb8..8c0e971f 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -50,7 +50,7 @@ fun App(cameraService: CameraService, attachmentDirectory: String) { if (BuildConfig.SUPABASE_ATTACHMENT_BUCKET != "null") { AttachmentQueue( db = db, remoteStorage = SupabaseRemoteStorage(supabase), - attachmentDirectory = attachmentDirectory, + attachmentsDirectory = attachmentDirectory, watchedAttachments = db.watch( "SELECT photo_id from todos WHERE photo_id IS NOT NULL" ) { diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt index ebb2e557..0ca8cfa5 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt @@ -77,7 +77,7 @@ internal class Todo( fun onItemDeleteClicked(item: TodoItem) { viewModelScope.launch { if (item.photoId != null) { - attachmentsQueue?.deleteFile(item.photoId) + attachmentsQueue?.deleteFile(item.photoId) {_,_ -> } } db.writeTransaction { tx -> tx.execute("DELETE FROM $TODOS_TABLE WHERE id = ?", listOf(item.id)) From fe0e67ee4e11bcb8dca7483fd3ef4018e6c103be Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 8 Apr 2025 11:28:42 +0200 Subject: [PATCH 20/51] cleanup --- core/README.md | 8 +- .../kotlin/com/powersync/AttachmentsTest.kt | 127 ++++++++++++++++-- .../powersync/attachments/AttachmentQueue.kt | 2 +- .../com/powersync/attachments/README.md | 10 +- .../com/powersync/db/PowerSyncDatabaseImpl.kt | 18 +-- 5 files changed, 146 insertions(+), 19 deletions(-) diff --git a/core/README.md b/core/README.md index db89fbb3..ee09bf8a 100644 --- a/core/README.md +++ b/core/README.md @@ -24,8 +24,14 @@ structure: ## Note on SQLDelight The PowerSync core module, internally makes use -of [SQLDelight](https://sqldelight.github.io/sqldelight/latest/) for it database API and typesafe database +of [SQLDelight](https://sqldelight.github.io/sqldelight/latest/) for it database API and typesafe +database query generation. The PowerSync core module does not currently support integrating with SQLDelight from client applications. + +## Attachment Helpers + +This module contains attachment helpers under the `com.powersync.attachments` package. See +the [Attachment Helpers README](./src/commonMain/kotlin/com/powersync/attachments/README.md) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 04420b8d..1e937a8c 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -16,6 +16,7 @@ import com.powersync.testutils.MockedRemoteStorage import com.powersync.testutils.UserRow import com.powersync.testutils.databaseTest import com.powersync.testutils.getTempDir +import com.powersync.testutils.waitFor import dev.mokkery.answering.throws import dev.mokkery.everySuspend import dev.mokkery.matcher.ArgMatchersScope @@ -29,11 +30,13 @@ import io.kotest.matchers.shouldNotBe import kotlinx.coroutines.flow.flowOf import kotlinx.io.files.Path import kotlin.test.Test +import kotlin.time.Duration.Companion.seconds @OptIn(ExperimentalKermitApi::class) class AttachmentsTest { fun watchAttachments(database: PowerSyncDatabase) = database.watch( + // language=SQL sql = """ SELECT @@ -75,6 +78,7 @@ class AttachmentsTest { // Monitor the attachments table for testing val attachmentQuery = database + // language=SQL .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } .testIn(this) @@ -109,6 +113,7 @@ class AttachmentsTest { // This code did not save an attachment before assigning a photo_id. // This is equivalent to requiring an attachment download database.execute( + // language=SQL """ INSERT INTO users (id, name, email, photo_id) @@ -143,6 +148,7 @@ class AttachmentsTest { // Now clear the user's photo_id. The attachment should be archived database.execute( + // language=SQL """ UPDATE users @@ -183,6 +189,7 @@ class AttachmentsTest { // Monitor the attachments table for testing val attachmentQuery = database + // language=SQL .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } .testIn(this) @@ -224,6 +231,7 @@ class AttachmentsTest { ) { tx, attachment -> // Set the photo_id of a new user to the attachment id tx.execute( + // language=SQL """ INSERT INTO users (id, name, email, photo_id) @@ -258,6 +266,7 @@ class AttachmentsTest { // Now clear the user's photo_id. The attachment should be archived database.execute( + // language=SQL """ UPDATE users @@ -281,6 +290,104 @@ class AttachmentsTest { } } + @Test + fun testAttachmentDelete() = + databaseTest { + turbineScope { + updateSchema(database) + val remote = spy(MockedRemoteStorage()) + + // Monitor the attachments table for testing + val attachmentQuery = + database + // language=SQL + .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } + .testIn(this) + + val queue = + AttachmentQueue( + db = database, + remoteStorage = remote, + attachmentsDirectory = getAttachmentsDir(), + watchedAttachments = watchAttachments(database), + /** + * Sets the cache limit to zero for this test. Archived records will + * immediately be deleted. + */ + archivedCacheLimit = 0, + syncThrottleDuration = 0.seconds, + ) + + doOnCleanup { + queue.stopSyncing() + queue.clearQueue() + queue.close() + attachmentQuery.cancel() + } + + queue.startSync() + + val result = attachmentQuery.awaitItem() + + // There should not be any attachment records here + result.size shouldBe 0 + + // Create an attachment (simulates a download) + database.execute( + // language=SQL + """ + INSERT INTO + users (id, name, email, photo_id) + VALUES + (uuid(), "steven", "steven@steven.com", uuid()) + """, + ) + + // language=SQL + val attachmentID = + database.get("SELECT photo_id FROM users") { it.getString("photo_id") } + + // Wait for the record to be synced (mocked backend will allow it) + waitFor { + val record = attachmentQuery.awaitItem().first() + record shouldNotBe null + record.state shouldBe AttachmentState.SYNCED.ordinal + } + + queue.deleteFile( + attachmentId = attachmentID, + ) { tx, attachment -> + tx.execute( + // language=SQL + """ + UPDATE + users + SET + photo_id = NULL + WHERE + photo_id = ? + """.trimIndent(), + listOf(attachment.id), + ) + } + + waitFor { + // Record should be deleted + val record = attachmentQuery.awaitItem().firstOrNull() + record shouldBe null + } + + // A delete should have been attempted for this file + verifySuspend { + remote.deleteFile( + any(), + ) + } + + attachmentQuery.cancel() + } + } + @Test fun testAttachmentCachedDownload() = databaseTest { @@ -292,6 +399,7 @@ class AttachmentsTest { // Monitor the attachments table for testing val attachmentQuery = database + // language=SQL .watch("SELECT * FROM attachments") { Attachment.fromCursor(it) } .testIn(this) @@ -325,6 +433,7 @@ class AttachmentsTest { // This code did not save an attachment before assigning a photo_id. // This is equivalent to requiring an attachment download database.execute( + // language=SQL """ INSERT INTO users (id, name, email, photo_id) @@ -359,11 +468,12 @@ class AttachmentsTest { // Now clear the user's photo_id. The attachment should be archived database.execute( + // language=SQL """ - UPDATE - users - SET - photo_id = NULL + UPDATE + users + SET + photo_id = NULL """, ) @@ -372,11 +482,12 @@ class AttachmentsTest { // Now if we set the photo_id, the archived record should be restored database.execute( + // language=SQL """ - UPDATE - users - SET - photo_id = ? + UPDATE + users + SET + photo_id = ? """, listOf(attachmentRecord.id), ) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index f403ab70..80c58069 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -449,7 +449,7 @@ public open class AttachmentQueue( ?: throw Exception("Attachment record with id $attachmentId was not found.") db.writeTransaction { tx -> - updateHook?.invoke(tx, attachment) + updateHook.invoke(tx, attachment) return@writeTransaction attachmentContext.upsertAttachment( attachment.copy(state = AttachmentState.QUEUED_DELETE.ordinal), tx, diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index d7c19e73..8fb62f7c 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -1,9 +1,17 @@ -# PowerSync Attachments +# PowerSync Attachment Helpers A [PowerSync](https://powersync.com) library to manage attachments in Kotlin Multiplatform apps. This package is included in the PowerSync Core module. +## Alpha Release + +Attachment helpers are currently in an alpha state, intended strictly for testing. Expect breaking +changes +and instability as development continues. + +Do not rely on this package for production use. + ## Usage An `AttachmentQueue` is used to manage and sync attachments in your app. The attachments' state are diff --git a/core/src/commonMain/kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt b/core/src/commonMain/kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt index 40973e81..23d7d368 100644 --- a/core/src/commonMain/kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt @@ -502,15 +502,17 @@ internal class PowerSyncDatabaseImpl( } override suspend fun close() = - mutex.withLock { - if (closed) { - return@withLock + runWrappedSuspending { + mutex.withLock { + if (closed) { + return@withLock + } + initializeJob.cancelAndJoin() + disconnectInternal() + internalDb.close() + resource.dispose() + closed = true } - initializeJob.cancelAndJoin() - disconnectInternal() - internalDb.close() - resource.dispose() - closed = true } /** From 8d19eedc0ab0d829e8a9964e4e3eba6f132e683e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 8 Apr 2025 11:29:18 +0200 Subject: [PATCH 21/51] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c634d0e8..8463a67e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Sqlite operation failure database is locked attempted to run migration and failed. closing connection ``` * Fix race condition causing data received during uploads not to be applied. +* Added Attachment helpers ## 1.0.0-BETA28 From f1c698a596276cd152c9c7a3d8ca97b77beda7a6 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 8 Apr 2025 11:44:58 +0200 Subject: [PATCH 22/51] cleanup --- .../java/com/powersync/androidexample/App.kt | 42 ++++++----- .../java/com/powersync/androidexample/Auth.kt | 7 +- .../powersync/androidexample/MainActivity.kt | 4 +- .../androidexample/SupabaseRemoteStorage.kt | 3 +- .../androidexample/components/EditDialog.kt | 69 ++++++++++--------- .../androidexample/powersync/Todo.kt | 39 ++++++----- .../androidexample/screens/TodosScreen.kt | 19 ++--- .../androidexample/ui/CameraService.kt | 49 ++++++------- settings.gradle.kts | 1 - 9 files changed, 126 insertions(+), 107 deletions(-) diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index 8c0e971f..ac26ebe2 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -32,7 +32,10 @@ import com.powersync.demos.screens.TodosScreen import kotlinx.coroutines.runBlocking @Composable -fun App(cameraService: CameraService, attachmentDirectory: String) { +fun App( + cameraService: CameraService, + attachmentDirectory: String, +) { val driverFactory = rememberDatabaseDriverFactory() val supabase = remember { @@ -49,24 +52,31 @@ fun App(cameraService: CameraService, attachmentDirectory: String) { remember { if (BuildConfig.SUPABASE_ATTACHMENT_BUCKET != "null") { AttachmentQueue( - db = db, remoteStorage = SupabaseRemoteStorage(supabase), + db = db, + remoteStorage = SupabaseRemoteStorage(supabase), attachmentsDirectory = attachmentDirectory, - watchedAttachments = db.watch( - "SELECT photo_id from todos WHERE photo_id IS NOT NULL" - ) { - WatchedAttachmentItem( - id = it.getString("photo_id"), - fileExtension = "jpg" - ) - }) - } else {null} } + watchedAttachments = + db.watch( + "SELECT photo_id from todos WHERE photo_id IS NOT NULL", + ) { + WatchedAttachmentItem( + id = it.getString("photo_id"), + fileExtension = "jpg", + ) + }, + ) + } else { + null + } + } val syncStatus = db.currentStatus val status by syncStatus.asFlow().collectAsState(syncStatus) - val navController = remember { - NavController(Screen.Home) - } + val navController = + remember { + NavController(Screen.Home) + } val authViewModel = remember { @@ -149,8 +159,8 @@ fun App(cameraService: CameraService, attachmentDirectory: String) { onTextChanged = todos.value::onEditorTextChanged, onDoneChanged = todos.value::onEditorDoneChanged, onPhotoClear = todos.value::onPhotoDelete, - onPhotoCapture = {todos.value::onPhotoCapture.invoke(cameraService)}, - attachmentsSupported = attachments != null + onPhotoCapture = { todos.value::onPhotoCapture.invoke(cameraService) }, + attachmentsSupported = attachments != null, ) } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt index 1b62d9c4..598aa886 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/Auth.kt @@ -22,7 +22,7 @@ internal class AuthViewModel( private val supabase: SupabaseConnector, private val db: PowerSyncDatabase, private val navController: NavController, - private val attachmentsQueue: AttachmentQueue? + private val attachmentsQueue: AttachmentQueue?, ) : ViewModel() { private val _authState = MutableStateFlow(AuthState.SignedOut) val authState: StateFlow = _authState @@ -38,8 +38,9 @@ internal class AuthViewModel( _userId.value = it.session.user?.id db.connect(supabase) attachmentsQueue?.startSync() - if (navController.currentScreen.value is Screen.SignIn - || navController.currentScreen.value is Screen.SignUp) { + if (navController.currentScreen.value is Screen.SignIn || + navController.currentScreen.value is Screen.SignUp + ) { navController.navigate(Screen.Home) } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt index b6a4373a..7f2cef02 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/MainActivity.kt @@ -9,6 +9,7 @@ import com.powersync.demos.App class MainActivity : ComponentActivity() { private val cameraService = CameraService(this) + override fun onCreate(savedInstanceState: Bundle?) { installSplashScreen() @@ -17,9 +18,8 @@ class MainActivity : ComponentActivity() { setContent { App( cameraService = cameraService, - attachmentDirectory = "${applicationContext.filesDir.canonicalPath}/attachments" + attachmentDirectory = "${applicationContext.filesDir.canonicalPath}/attachments", ) } } - } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt index 925e2302..22858914 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt @@ -24,7 +24,8 @@ class SupabaseRemoteStorage( connector.storageBucket().upload(attachment.filename, buffer) } - override suspend fun downloadFile(attachment: Attachment): Flow = flowOf(connector.storageBucket().downloadAuthenticated(attachment.filename)) + override suspend fun downloadFile(attachment: Attachment): Flow = + flowOf(connector.storageBucket().downloadAuthenticated(attachment.filename)) override suspend fun deleteFile(attachment: Attachment) { connector.storageBucket().delete(attachment.filename) diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt index 7a51ac37..6c4785db 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/components/EditDialog.kt @@ -39,7 +39,7 @@ internal fun EditDialog( onDoneChanged: (Boolean) -> Unit, onPhotoClear: () -> Unit, onPhotoCapture: () -> Unit, - attachmentsSupported: Boolean = false + attachmentsSupported: Boolean = false, ) { EditDialog( onCloseRequest = onCloseClicked, @@ -62,37 +62,39 @@ internal fun EditDialog( ) } - val bitmap = remember(item.photoURI) { - item.photoURI?.let { BitmapFactory.decodeFile(it)?.asImageBitmap() } - } + val bitmap = + remember(item.photoURI) { + item.photoURI?.let { BitmapFactory.decodeFile(it)?.asImageBitmap() } + } if (attachmentsSupported == true) { - Box( - modifier = Modifier - .clickable { if (item.photoId == null) onPhotoCapture() } - .padding(8.dp), - contentAlignment = Alignment.Center - ) { - if (bitmap == null) { - Button( - onClick = onPhotoCapture, - modifier = Modifier.align(Alignment.Center), - contentPadding = PaddingValues(0.dp) - ) { - Text("Add Photo", color = Color.Gray) - } - } else { - Image(bitmap = bitmap, contentDescription = "Photo Preview") - Button( - onClick = onPhotoClear, - modifier = Modifier.align(Alignment.TopEnd), - contentPadding = PaddingValues(0.dp) - ) { - Text("Clear Photo", color = Color.Red) + Box( + modifier = + Modifier + .clickable { if (item.photoId == null) onPhotoCapture() } + .padding(8.dp), + contentAlignment = Alignment.Center, + ) { + if (bitmap == null) { + Button( + onClick = onPhotoCapture, + modifier = Modifier.align(Alignment.Center), + contentPadding = PaddingValues(0.dp), + ) { + Text("Add Photo", color = Color.Gray) + } + } else { + Image(bitmap = bitmap, contentDescription = "Photo Preview") + Button( + onClick = onPhotoClear, + modifier = Modifier.align(Alignment.TopEnd), + contentPadding = PaddingValues(0.dp), + ) { + Text("Clear Photo", color = Color.Red) + } } } } - } } } } @@ -100,16 +102,17 @@ internal fun EditDialog( @Composable private fun EditDialog( onCloseRequest: () -> Unit, - content: @Composable () -> Unit + content: @Composable () -> Unit, ) { Dialog( onDismissRequest = onCloseRequest, ) { - Card(elevation = CardDefaults.cardElevation(defaultElevation = 8.dp) ) { + Card(elevation = CardDefaults.cardElevation(defaultElevation = 8.dp)) { Column( - modifier = Modifier - .padding(8.dp) - .height(IntrinsicSize.Min) + modifier = + Modifier + .padding(8.dp) + .height(IntrinsicSize.Min), ) { ProvideTextStyle(MaterialTheme.typography.bodySmall) { Text(text = "Edit todo") @@ -125,7 +128,7 @@ private fun EditDialog( Button( onClick = onCloseRequest, - modifier = Modifier.align(Alignment.End) + modifier = Modifier.align(Alignment.End), ) { Text(text = "Done") } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt index 0ca8cfa5..5e141e29 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/powersync/Todo.kt @@ -1,6 +1,5 @@ package com.powersync.demos.powersync -import androidx.activity.result.contract.ActivityResultContracts import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import co.touchlab.kermit.Logger @@ -53,7 +52,7 @@ internal class Todo( completed = cursor.getLongOptional("completed") == 1L, listId = cursor.getString("list_id"), photoId = cursor.getStringOptional("photo_id"), - photoURI = cursor.getStringOptional("local_uri") + photoURI = cursor.getStringOptional("local_uri"), ) } @@ -77,7 +76,7 @@ internal class Todo( fun onItemDeleteClicked(item: TodoItem) { viewModelScope.launch { if (item.photoId != null) { - attachmentsQueue?.deleteFile(item.photoId) {_,_ -> } + attachmentsQueue?.deleteFile(item.photoId) { _, _ -> } } db.writeTransaction { tx -> tx.execute("DELETE FROM $TODOS_TABLE WHERE id = ?", listOf(item.id)) @@ -96,7 +95,7 @@ internal class Todo( } viewModelScope.launch { - db.writeTransaction { tx -> + db.writeTransaction { tx -> tx.execute( "INSERT INTO $TODOS_TABLE (id, created_at, created_by, description, list_id) VALUES (uuid(), datetime(), ?, ?, ?)", listOf(userId, _inputText.value, listId), @@ -134,31 +133,33 @@ internal class Todo( fun onPhotoCapture(cameraService: CameraService) { viewModelScope.launch { val item = requireNotNull(_editingItem.value) - val photoData = try { - cameraService.takePicture() - } catch (ex: Exception) { - if (ex is CancellationException) { - throw ex - } else { - // otherwise ignore - return@launch + val photoData = + try { + cameraService.takePicture() + } catch (ex: Exception) { + if (ex is CancellationException) { + throw ex + } else { + // otherwise ignore + return@launch + } + } + val attachment = + attachmentsQueue!!.saveFile(data = flowOf(photoData), mediaType = "image/jped", fileExtension = "jpg") { tx, attachment -> + tx.execute("UPDATE $TODOS_TABLE SET photo_id = ? WHERE id = ?", listOf(attachment.id, item.id)) } - } - val attachment = attachmentsQueue!!.saveFile(data = flowOf(photoData), mediaType = "image/jped", fileExtension = "jpg" ) {tx, attachment -> - tx.execute("UPDATE $TODOS_TABLE SET photo_id = ? WHERE id = ?", listOf(attachment.id, item.id)) - } - updateEditingItem(item = item) {it.copy(photoURI = attachment.localUri)} + updateEditingItem(item = item) { it.copy(photoURI = attachment.localUri) } } } fun onPhotoDelete() { viewModelScope.launch { val item = requireNotNull(_editingItem.value) - attachmentsQueue!!.deleteFile(item.photoId!!) {tx, _ -> + attachmentsQueue!!.deleteFile(item.photoId!!) { tx, _ -> tx.execute("UPDATE $TODOS_TABLE SET photo_id = NULL WHERE id = ?", listOf(item.id)) } - updateEditingItem(item = item) {it.copy(photoURI = null)} + updateEditingItem(item = item) { it.copy(photoURI = null) } } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/screens/TodosScreen.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/screens/TodosScreen.kt index 403adfac..4cb0508b 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/screens/TodosScreen.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/screens/TodosScreen.kt @@ -6,13 +6,13 @@ import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.width +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.automirrored.filled.ArrowBack +import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.Icon import androidx.compose.material3.IconButton import androidx.compose.material3.Text import androidx.compose.material3.TopAppBar -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.automirrored.filled.ArrowBack -import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier import androidx.compose.ui.text.style.TextAlign @@ -36,7 +36,7 @@ internal fun TodosScreen( onItemDoneChanged: (item: TodoItem, isDone: Boolean) -> Unit, onItemDeleteClicked: (item: TodoItem) -> Unit, onAddItemClicked: () -> Unit, - onInputTextChanged: (value: String) -> Unit + onInputTextChanged: (value: String) -> Unit, ) { Column(modifier) { TopAppBar( @@ -44,8 +44,9 @@ internal fun TodosScreen( Text( "Todo List", textAlign = TextAlign.Center, - modifier = Modifier.fillMaxWidth().padding(end = 36.dp) - ) }, + modifier = Modifier.fillMaxWidth().padding(end = 36.dp), + ) + }, navigationIcon = { IconButton(onClick = { navController.navigate(Screen.Home) }) { Icon(Icons.AutoMirrored.Filled.ArrowBack, contentDescription = "Go back") @@ -54,14 +55,14 @@ internal fun TodosScreen( actions = { WifiIcon(isConnected ?: false) Spacer(modifier = Modifier.width(16.dp)) - } + }, ) Input( text = inputText, onAddClicked = onAddItemClicked, onTextChanged = onInputTextChanged, - screen = Screen.Todos + screen = Screen.Todos, ) Box(Modifier.weight(1F)) { @@ -69,7 +70,7 @@ internal fun TodosScreen( items = items, onItemClicked = onItemClicked, onItemDoneChanged = onItemDoneChanged, - onItemDeleteClicked = onItemDeleteClicked + onItemDeleteClicked = onItemDeleteClicked, ) } } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/ui/CameraService.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/ui/CameraService.kt index 2a78e23a..4057c8d1 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/ui/CameraService.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/ui/CameraService.kt @@ -16,37 +16,40 @@ import java.util.Locale /** * A very basic camera service. This should not be used in production. */ -class CameraService(val activity: MainActivity) { +class CameraService( + val activity: MainActivity, +) { private var currentPhotoUri: Uri? = null private var pictureResult: CompletableDeferred? = null private var file: File? = null private val mutex = Mutex() - private val takePictureLauncher = activity.registerForActivityResult(ActivityResultContracts.TakePicture()) { success -> - if (success && currentPhotoUri != null) { - activity.contentResolver.openInputStream(currentPhotoUri!!)?.use { - pictureResult!!.complete(it.readBytes()) + private val takePictureLauncher = + activity.registerForActivityResult(ActivityResultContracts.TakePicture()) { success -> + if (success && currentPhotoUri != null) { + activity.contentResolver.openInputStream(currentPhotoUri!!)?.use { + pictureResult!!.complete(it.readBytes()) + } + file!!.delete() + } else { + pictureResult!!.completeExceptionally(Exception("Could not capture photo")) } - file!!.delete() - } else { - pictureResult!!.completeExceptionally(Exception("Could not capture photo")) + file = null + currentPhotoUri = null + pictureResult = null } - file = null - currentPhotoUri = null - pictureResult = null - } + suspend fun takePicture(): ByteArray = + mutex.withLock { + pictureResult = CompletableDeferred() + val timeStamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()).format(Date()) + val storageDir = activity.getExternalFilesDir(Environment.DIRECTORY_PICTURES) + file = File.createTempFile("JPEG_${timeStamp}_", ".jpg", storageDir) + currentPhotoUri = FileProvider.getUriForFile(activity, "${activity.packageName}.fileprovider", file!!) - suspend fun takePicture(): ByteArray = mutex.withLock { - pictureResult = CompletableDeferred() - val timeStamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()).format(Date()) - val storageDir = activity.getExternalFilesDir(Environment.DIRECTORY_PICTURES) - file = File.createTempFile("JPEG_${timeStamp}_", ".jpg", storageDir) - currentPhotoUri = FileProvider.getUriForFile(activity, "${activity.packageName}.fileprovider", file!!) + takePictureLauncher.launch(currentPhotoUri!!) - takePictureLauncher.launch(currentPhotoUri!!) - - return pictureResult!!.await() - } -} \ No newline at end of file + return pictureResult!!.await() + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 1e529c5a..b7472215 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -27,6 +27,5 @@ include(":persistence") include(":PowerSyncKotlin") include(":compose") -include(":attachments") enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") From 2da25d67d39ce69a77c86093caefcb56613dde72 Mon Sep 17 00:00:00 2001 From: benitav Date: Wed, 16 Apr 2025 11:57:34 +0200 Subject: [PATCH 23/51] Restructure the attachments Readme with some simplified wording --- .../com/powersync/attachments/README.md | 340 ++++++++---------- 1 file changed, 143 insertions(+), 197 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index 8fb62f7c..f1cc7094 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -1,36 +1,36 @@ # PowerSync Attachment Helpers -A [PowerSync](https://powersync.com) library to manage attachments in Kotlin Multiplatform apps. +A [PowerSync](https://powersync.com) library to manage attachments (such as images or files) in Kotlin Multiplatform apps. This package is included in the PowerSync Core module. +A concrete example implementation of these attachment helpers is available in the [Android Supabase Demo](/demos/android-supabase-todolist/README.md) app. + ## Alpha Release -Attachment helpers are currently in an alpha state, intended strictly for testing. Expect breaking -changes -and instability as development continues. +Attachment helpers are currently in an alpha state, intended strictly for testing. Expect breaking changes and instability as development continues. Do not rely on this package for production use. ## Usage An `AttachmentQueue` is used to manage and sync attachments in your app. The attachments' state are -stored in a local only attachments table. +stored in a local-only attachments table. ### Key Assumptions -- Each attachment should be identifiable by a unique ID. -- Attachments are immutable. -- Relational data should contain a foreign key column that references the attachment ID. +- Each attachment is identified by a unique ID +- Attachments are immutable once created +- Relational data should reference attachments using a foreign key column - Relational data should reflect the holistic state of attachments at any given time. An existing local attachment will deleted locally if no relational data references it. -### Example +### Example Implementation In this example, the user captures photos when checklist items are completed as part of an inspection workflow. -The schema for the `checklist` table: +1. First, define your schema including the `checklist` table and the local-only attachments table: ```kotlin val checklists = Table( @@ -45,110 +45,72 @@ val checklists = Table( val schema = Schema( UserRow.table, - // Includes the table which stores attachment states + // Add the local-only table which stores attachment states + // Learn more about this function below createAttachmentsTable("attachments") ) ``` -The `createAttachmentsTable` function defines the local only attachment state storage table. - -An attachments table definition can be created with the following options. - -| Option | Description | Default | -|--------|-----------------------|---------------| -| `name` | The name of the table | `attachments` | - -The default columns in `AttachmentTable`: - -| Column Name | Type | Description | -|--------------|-----------|--------------------------------------------------------------------------------------------------------------------| -| `id` | `TEXT` | The ID of the attachment record | -| `filename` | `TEXT` | The filename of the attachment | -| `media_type` | `TEXT` | The media type of the attachment | -| `state` | `INTEGER` | The state of the attachment, one of `AttachmentState` enum values | -| `timestamp` | `INTEGER` | The timestamp of last update to the attachment record | -| `size` | `INTEGER` | The size of the attachment in bytes | -| `has_synced` | `INTEGER` | Internal tracker which tracks if the attachment has ever been synced. This is used for caching/archiving purposes. | -| `meta_data` | `TEXT` | Any extra meta data for the attachment. JSON is usually a good choice. | - -### Steps to implement - -1. Create an instance of `AttachmentQueue` from `com.powersync.attachments`. This class provides - default syncing utilities and implements a default sync strategy. This class is open and can be - overridden for custom functionality. +2. Create an `AttachmentQueue` instance. This class provides +default syncing utilities and implements a default sync strategy. This class is open and can be overridden for custom functionality: ```kotlin - val queue = AttachmentQueue( db = db, attachmentsDirectory = attachmentsDirectory, remoteStorage = SupabaseRemoteStorage(supabase), watchedAttachments = db.watch( - sql = - """ - SELECT - photo_id - FROM - checklists - WHERE - photo_id IS NOT NULL - """, + sql = """ + SELECT photo_id + FROM checklists + WHERE photo_id IS NOT NULL + """, ) { WatchedAttachmentItem(id = it.getString("photo_id"), fileExtension = "jpg") } ) ``` -* The `attachmentsDirectory`, specifies where local attachment - files should be stored. This directory needs to be provided to the constructor. On Android +* The `attachmentsDirectory`, specifies where local attachment files should be stored. This directory needs to be provided to the constructor. On Android `"${applicationContext.filesDir.canonicalPath}/attachments"` is a good choice. -* The `remoteStorage` is responsible for connecting to the attachments backend. See the - `RemoteStorageAdapter` interface +* The `remoteStorage` is responsible for connecting to the attachments backend. See the `RemoteStorageAdapter` interface definition [here](https://github.com/powersync-ja/powersync-kotlin/blob/main/core/src/commonMain/kotlin/com.powersync/attachments/RemoteStorageAdapter.ts). -* `watchAttachments` is a `Flow` of `WatchedAttachmentItem`. - The `WatchedAttachmentItem`s represent the attachments which should be present in the - application. We recommend using `PowerSync`'s `watch` query as shown above. In this example we - provide the `fileExtension` for all photos. This information could also be +* `watchAttachments` is a `Flow` of `WatchedAttachmentItem`. The `WatchedAttachmentItem`s represent the attachments which should be present in the + application. We recommend using `PowerSync`'s `watch` query as shown above. In this example we provide the `fileExtension` for all photos. This information could also be obtained from the query if necessary. -2. Implement a `RemoteStorageAdapter` which interfaces with a remote storage provider. This will be - used for downloading, uploading and deleting attachments. +3. Implement a `RemoteStorageAdapter` which interfaces with a remote storage provider. This will be + used for downloading, uploading and deleting attachments: ```kotlin val remote = object : RemoteStorage() { - override suspend fun uploadFile( - fileData: Flow, - attachment: Attachment, - ) { - TODO("Make a request to the backend") + override suspend fun uploadFile(fileData: Flow, attachment: Attachment) { + TODO("Implement upload to your backend") } override suspend fun downloadFile(attachment: Attachment): Flow { - TODO("Make a request to the backend") + TODO("Implement download from your backend") } override suspend fun deleteFile(attachment: Attachment) { - TODO("Make a request to the backend") + TODO("Implement delete in your backend") } - } - ``` -3. Call `startSync()` to start syncing attachments. +4. Start the sync process: ```kotlin queue.startSync() ``` -4. Finally, to create an attachment and add it to the queue, call `saveFile()`. This method will +5. Create and save attachments using `saveFile()`. This method will save the file to the local storage, create an attachment record which queues the file for upload - to the remote storage and allows assigning the newly created attachment ID to a checklist item. + to the remote storage and allows assigning the newly created attachment ID to a checklist item: ```kotlin queue.saveFile( - // The attachment's data flow, this is just an example - data = flowOf(ByteArray(1)), + data = flowOf(ByteArray(1)), // Your attachment data mediaType = "image/jpg", fileExtension = "jpg", ) { tx, attachment -> @@ -158,24 +120,120 @@ queue.saveFile( */ tx.execute( """ - UPDATE - checklists - SET - photo_id = ? - WHERE - id = ? + UPDATE checklists + SET photo_id = ? + WHERE id = ? """, listOf(attachment.id, checklistId), ) } ``` -#### Handling Errors +## Implementation Details + +### Attachment Table Structure + +The `createAttachmentsTable` function creates a local-only table for tracking attachment states. + +An attachments table definition can be created with the following options: + +| Option | Description | Default | +|--------|-----------------------|---------------| +| `name` | The name of the table | `attachments` | + +The default columns are: + +| Column Name | Type | Description | +|--------------|-----------|--------------------------------------------------------------------------------------------------------------------| +| `id` | `TEXT` | Unique identifier for the attachment | +| `filename` | `TEXT` | The filename of the attachment | +| `media_type` | `TEXT` | The media type of the attachment | +| `state` | `INTEGER` | Current state of the attachment (see `AttachmentState` enum) | +| `timestamp` | `INTEGER` | The timestamp of last update to the attachment | +| `size` | `INTEGER` | File size in bytes | +| `has_synced` | `INTEGER` | Internal flag tracking if the attachment has ever been synced (used for caching) | +| `meta_data` | `TEXT` | Additional metadata in JSON format | + +### Attachment States + +Attachments are managed through the following states: + +| State | Description | +|-------------------|-------------------------------------------------------------------------------| +| `QUEUED_UPLOAD` | Attachment is queued for upload to cloud storage | +| `QUEUED_DELETE` | Attachment is queued for deletion from cloud storage and local storage | +| `QUEUED_DOWNLOAD` | Attachment is queued for download from cloud storage | +| `SYNCED` | Attachment is fully synced | +| `ARCHIVED` | Attachment is orphaned - i.e. no longer referenced by any data | + +### Sync Process + +The `AttachmentQueue` implements a sync process with these components: + +1. **State Monitoring**: The queue watches the attachments table for records in `QUEUED_UPLOAD`, `QUEUED_DELETE`, and `QUEUED_DOWNLOAD` states. An event loop triggers calls to the remote storage for these operations. + +2. **Periodic Sync**: By default, the queue triggers a sync every 30 seconds to retry failed uploads/downloads, in particular after the app was offline. This interval can be configured by setting `syncInterval` in the `AttachmentQueue` constructor options, or disabled by setting the interval to `0`. + +3. **Watching State**: The `watchedAttachments` flow in the `AttachmentQueue` constructor is used to maintain consistency between local and remote states: + - New items trigger downloads - see the Download Process below. + - Missing items trigger archiving - see Cache Management below. + +### Upload Process + +The `saveFile` method handles attachment creation and upload: + +1. The attachment is saved to local storage +2. An `AttachmentRecord` is created with `QUEUED_UPLOAD` state, linked to the local file using `localURI` +3. The attachment must be assigned to relational data in the same transaction, since this data is constantly watched and should always represent the attachment queue state +4. The `RemoteStorage` `uploadFile` function is called +5. On successful upload, the state changes to `SYNCED` +6. If upload fails, the record stays in `QUEUED_UPLOAD` state for retry + +### Download Process + +Attachments are scheduled for download when the `watchedAttachments` flow emits a new item that is not present locally: + +1. An `AttachmentRecord` is created with `QUEUED_DOWNLOAD` state +2. The `RemoteStorage` `downloadFile` function is called +3. The received data is saved to local storage +4. On successful download, the state changes to `SYNCED` +5. If download fails, the operation is retried in the next sync cycle + +### Delete Process + +The `deleteFile` method deletes attachments from both local and remote storage: + +1. The attachment record moves to `QUEUED_DELETE` state +2. The attachment must be unassigned from relational data in the same transaction, since this data is constantly watched and should always represent the attachment queue state +3. On successful deletion, the record is removed +4. If deletion fails, the operation is retried in the next sync cycle -The attachment queue automatically retries failed sync operations. Retries continue indefinitely -until success. A `SyncErrorHanlder` can be provided to the `AttachmentQueue` constructor. This -handler provides methods which are invoked on a remote sync exception. The handler can return a -Boolean which indicates if the attachment sync should be retried or archived. +### Cache Management + +The `AttachmentQueue` implements a caching system for archived attachments: + +1. Local attachments are marked as `ARCHIVED` if the `watchedAttachments` flow no longer references them +2. Archived attachments are kept in the cache for potential future restoration +3. The cache size is controlled by the `archivedCacheLimit` parameter in the `AttachmentQueue` constructor +4. By default, the queue keeps the last 100 archived attachment records +5. When the cache limit is reached, the oldest archived attachments are permanently deleted +6. If an archived attachment is referenced again while still in the cache, it can be restored +7. The cache limit can be configured in the `AttachmentQueue` constructor + +### Error Handling + +1. **Automatic Retries**: + - Failed uploads/downloads/deletes are automatically retried + - The sync interval (default 30 seconds) ensures periodic retry attempts + - Retries continue indefinitely until successful + +2. **Custom Error Handling**: + - A `SyncErrorHandler` can be implemented to customize retry behavior (see example below) + - The handler can decide whether to retry or archive failed operations + - Different handlers can be provided for upload, download, and delete operations + + +Example of a custom `SyncErrorHandler`: ```kotlin val errorHandler = object : SyncErrorHandler { @@ -199,122 +257,10 @@ val errorHandler = object : SyncErrorHandler { ): Boolean { TODO("Return if the attachment sync should be retried") } - } -// Pass the handler to the queue constructor val queue = AttachmentQueue( -// ..., - errorHandler = errorHandler, -// ... + // ... other parameters ... + errorHandler = errorHandler ) -``` - -# Implementation details - -## Attachment State - -The `AttachmentQueue` class manages attachments in your app by tracking their state. - -The state of an attachment can be one of the following: - -| State | Description | -|-------------------|-------------------------------------------------------------------------------| -| `QUEUED_UPLOAD` | The attachment has been queued for upload to the cloud storage | -| `QUEUED_DELETE` | The attachment has been queued for delete in the cloud storage (and locally) | -| `QUEUED_DOWNLOAD` | The attachment has been queued for download from the cloud storage | -| `SYNCED` | The attachment has been synced | -| `ARCHIVED` | The attachment has been orphaned, i.e. the associated record has been deleted | - -## Syncing attachments - -The `AttachmentQueue` sets a watched query on the `attachments` table, for record in the -`QUEUED_UPLOAD`, `QUEUED_DELETE` and `QUEUED_DOWNLOAD` state. An event loop triggers calls to the -remote storage for these operations. - -In addition to watching for changes, the `AttachmentQueue` also triggers a sync periodically. -This will retry any failed uploads/downloads, in particular after the app was offline. By default, -this is every 30 seconds, but can be configured by setting `syncInterval` in the -`AttachmentQueue` constructor options, or disabled by setting the interval to `0`. - -### Watching State - -The `watchedAttachments` flow provided to the `AttachmentQueue` constructor is used to reconcile the -local Attachment state. Each emission of the flow should represent the current attachment state. The -updated state is constantly compared to the current queue state. Items are queued based off the -difference. - -* A new watched item which is not present in the current queue is treated as an upstream Attachment - creation which needs to be downloaded. - * An attachment record is create using the provided watched item. The filename will be inferred - using a default filename resolver if it has not been provided in the watched item. - * The syncing service will attempt to download the attachment from the remote storage. - * The attachment will be saved to the local filesystem. The `local_uri` on the attachment record - will be updated. - * The attachment state will be updated to `SYNCED` -* Local attachments are archived if the watched state no longer includes the item. Archived items - are cached and can be restored if the watched state includes them in future. The number of cached - items is defined by the `archivedCacheLimit` parameter in the `AttachmentQueue` constructor. Items - are deleted once the cache limit is reached. - -### Uploading - -The `saveFile` method provides a simple method for creating attachments which should be uploaded to -the backend. This method accepts the raw file content and meta data. This function: - -* Persists the attachment to the local filesystem -* Creates an attachment record linked to the local attachment file. -* Queues the attachment for upload. -* Allows assigning the attachment to relational data. - * It's important to assign the attachment to relational data since this data is constantly - watched and should always represent the attachment queue state. Failure to assign the - attachment could result in a failed upload. The attachment record will be archived. - -The sync process after calling `saveFile` is: - -- An `AttachmentRecord` is created or updated with a state of `QUEUED_UPLOAD`. -- The `RemoteStorage` `uploadFile` function is called with the `Attachment` record. -- The `AttachmentQueue` picks this up and upon successful upload to the remote storage, sets the - state to `SYNCED`. -- If the upload is not successful, the record remains in `QUEUED_UPLOAD` state and uploading will be - retried when syncing triggers again. Retries can be stopped by providing an `errorHandler`. - -### Downloading - -Attachments are schedules for download when the `watchedAttachments` flow emits a -`WatchedAttachmentItem` which is not present in the queue. - -- An `AttachmentRecord` is created or updated with `QUEUED_DOWNLOAD` state. -- The `RemoteStorage` `downloadFile` function is called with the attachment record. -- The received data is persisted to the local filesystem. -- If this is successful, update the `AttachmentRecord` state to `SYNCED`. -- If any of these fail, the download is retried in the next sync trigger. - -### Deleting attachments - -Local attachments are archived and deleted (locally) if the `watchedAttachments` flow no longer -references them. Archived attachments are deleted locally after cache invalidation. - -In some cases users might want to explicitly delete an attachment in the backend. The `deleteFile` -function provides a mechanism for this. This function: - -* Deletes the attachment on the local filesystem -* Updates the record to the `QUEUED_DELETE` state -* Allows removing assignments to relational data. - * It's important to unassign the attachment from relational data since this data is constantly - watched and should always represent the attachment queue state. Failure to unassign the - attachment could result in a failed delete. - -### Expire Cache - -When PowerSync removes a record, as a result of coming back online or conflict resolution for -instance: - -- Any associated `AttachmentRecord` is orphaned. -- On the next sync trigger, the `AttachmentQueue` sets all records that are orphaned to `ARCHIVED` - state. -- By default, the `AttachmentQueue` only keeps the last `100` attachment records and then expires - the rest. -- In some cases these records (attachment ids) might be restored. An archived attachment will be - restored if it is still in the cache. This can be configured by setting `cacheLimit` in the - `AttachmentQueue` constructor options. +``` \ No newline at end of file From 2255da2ddb22838524f0682d367565878963ef43 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 16:30:25 +0200 Subject: [PATCH 24/51] check if file exists before delete --- .../kotlin/com/powersync/attachments/sync/SyncingService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index 46cb920b..36216fe1 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -258,7 +258,7 @@ public class SyncingService( private suspend fun deleteAttachment(attachment: Attachment): Attachment { try { remoteStorage.deleteFile(attachment) - if (attachment.localUri != null) { + if (attachment.localUri != null && localStorage.fileExists(attachment.localUri)) { localStorage.deleteFile(attachment.localUri) } return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) From 48b3441aff0ed070d2f1d19950e1459faaf828cb Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 16:45:53 +0200 Subject: [PATCH 25/51] Move Supabase Remote Storage to Connectors module --- .../supabase/SupabaseRemoteStorage.kt | 57 +++++++++++++++++++ .../java/com/powersync/androidexample/App.kt | 2 +- .../androidexample/SupabaseRemoteStorage.kt | 33 ----------- 3 files changed, 58 insertions(+), 34 deletions(-) create mode 100644 connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseRemoteStorage.kt delete mode 100644 demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt diff --git a/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseRemoteStorage.kt b/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseRemoteStorage.kt new file mode 100644 index 00000000..5c3661af --- /dev/null +++ b/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseRemoteStorage.kt @@ -0,0 +1,57 @@ +package com.powersync.connector.supabase + +import com.powersync.attachments.Attachment +import com.powersync.attachments.RemoteStorage +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flowOf + +/** + * Implementation of [RemoteStorage] that uses Supabase as the backend storage provider. + * + * @property connector The Supabase connector used to interact with the Supabase storage bucket. + */ +public class SupabaseRemoteStorage( + public val connector: SupabaseConnector, +) : RemoteStorage { + + /** + * Uploads a file to the Supabase storage bucket. + * + * @param fileData A [Flow] of [ByteArray] representing the file data to be uploaded. + * @param attachment The [Attachment] metadata associated with the file. + * @throws IllegalStateException If the attachment size is not specified. + */ + override suspend fun uploadFile( + fileData: Flow, + attachment: Attachment, + ) { + val byteSize = attachment.size?.toInt() ?: error("Cannot upload a file with no byte size specified") + // Supabase wants a single ByteArray + val buffer = ByteArray(byteSize) + var position = 0 + fileData.collect { + it.copyInto(buffer, destinationOffset = position) + position += it.size + } + + connector.storageBucket().upload(attachment.filename, buffer) + } + + /** + * Downloads a file from the Supabase storage bucket. + * + * @param attachment The [Attachment] record associated with the file to be downloaded. + * @return A [Flow] of [ByteArray] representing the file data. + */ + override suspend fun downloadFile(attachment: Attachment): Flow = + flowOf(connector.storageBucket().downloadAuthenticated(attachment.filename)) + + /** + * Deletes a file from the Supabase storage bucket. + * + * @param attachment The [Attachment] record associated with the file to be deleted. + */ + override suspend fun deleteFile(attachment: Attachment) { + connector.storageBucket().delete(attachment.filename) + } +} \ No newline at end of file diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index ac26ebe2..a8b76577 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -13,12 +13,12 @@ import androidx.compose.runtime.rememberUpdatedState import androidx.compose.ui.Modifier import com.powersync.PowerSyncDatabase import com.powersync.androidexample.BuildConfig -import com.powersync.androidexample.SupabaseRemoteStorage import com.powersync.androidexample.ui.CameraService import com.powersync.attachments.AttachmentQueue import com.powersync.attachments.WatchedAttachmentItem import com.powersync.compose.rememberDatabaseDriverFactory import com.powersync.connector.supabase.SupabaseConnector +import com.powersync.connector.supabase.SupabaseRemoteStorage import com.powersync.db.getString import com.powersync.demos.components.EditDialog import com.powersync.demos.powersync.ListContent diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt deleted file mode 100644 index 22858914..00000000 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/SupabaseRemoteStorage.kt +++ /dev/null @@ -1,33 +0,0 @@ -package com.powersync.androidexample - -import com.powersync.attachments.Attachment -import com.powersync.attachments.RemoteStorage -import com.powersync.connector.supabase.SupabaseConnector -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.flowOf - -class SupabaseRemoteStorage( - val connector: SupabaseConnector, -) : RemoteStorage { - override suspend fun uploadFile( - fileData: Flow, - attachment: Attachment, - ) { - // Supabase wants a single ByteArray - val buffer = ByteArray(attachment.size!!.toInt()) - var position = 0 - fileData.collect { - System.arraycopy(it, 0, buffer, position, it.size) - position += it.size - } - - connector.storageBucket().upload(attachment.filename, buffer) - } - - override suspend fun downloadFile(attachment: Attachment): Flow = - flowOf(connector.storageBucket().downloadAuthenticated(attachment.filename)) - - override suspend fun deleteFile(attachment: Attachment) { - connector.storageBucket().delete(attachment.filename) - } -} From 6dfedd5d2a60d97a7cb8651f31d52edfab0903c3 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 16:52:11 +0200 Subject: [PATCH 26/51] Use enum throughout for AttachmentState --- .../com/powersync/attachments/Attachment.kt | 89 ++++++++++++------- .../powersync/attachments/AttachmentQueue.kt | 18 ++-- .../attachments/sync/SyncingService.kt | 21 +++-- 3 files changed, 76 insertions(+), 52 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt index 3693f267..6d7c715f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt @@ -7,59 +7,80 @@ import com.powersync.db.getString import com.powersync.db.getStringOptional /** - * Enum for the attachment state + * Represents the state of an attachment. */ public enum class AttachmentState { + /** + * The attachment is queued for download from the remote storage. + */ QUEUED_DOWNLOAD, + + /** + * The attachment is queued for upload to the remote storage. + */ QUEUED_UPLOAD, + + /** + * The attachment is queued for deletion from the remote storage. + */ QUEUED_DELETE, + + /** + * The attachment is fully synchronized with the remote storage. + */ SYNCED, - ARCHIVED, + + /** + * The attachment is archived and no longer actively synchronized. + */ + ARCHIVED; + + public companion object { + /** + * Constructs an [AttachmentState] from the corresponding integer value. + * + * @param value The integer value representing the ordinal of the enum. + * @return The corresponding [AttachmentState]. + * @throws IllegalArgumentException If the value does not match any [AttachmentState]. + */ + public fun fromLong(value: Long): AttachmentState { + return entries.getOrNull(value.toInt()) + ?: throw IllegalArgumentException("Invalid value for AttachmentState: $value") + } + } } /** - * Data class representing an attachment + * Represents an attachment with metadata and state information. + * + * @property id Unique identifier for the attachment. + * @property timestamp Timestamp of the last record update. + * @property filename Name of the attachment file, e.g., `[id].jpg`. + * @property state Current state of the attachment, represented as an ordinal of [AttachmentState]. + * @property localUri Local URI pointing to the attachment file, if available. + * @property mediaType Media type of the attachment, typically represented as a MIME type. + * @property size Size of the attachment in bytes, if available. + * @property hasSynced Indicates whether the attachment has been synced locally before. + * @property metaData Additional metadata associated with the attachment. */ public data class Attachment( - /** - * Unique identifier - */ val id: String, - /** - * Timestamp for the last record update - */ val timestamp: Long = 0, - /** - * Attachment filename e.g. `[id].jpg` - */ val filename: String, - /** - * Current attachment state - */ - val state: Int = AttachmentState.QUEUED_DOWNLOAD.ordinal, - /** - * Local URI pointing to the attachment file - */ + val state: AttachmentState = AttachmentState.QUEUED_DOWNLOAD, val localUri: String? = null, - /** - * Attachment media type. Usually represented by a MIME type. - */ val mediaType: String? = null, - /** - * Attachment byte size - */ val size: Long? = null, - /** - * Specifies if the attachment has been synced locally before. This is particularly useful - * for restoring archived attachments in edge cases. - */ val hasSynced: Int = 0, - /** - * Extra attachment meta data. - */ val metaData: String? = null, ) { public companion object { + /** + * Creates an [Attachment] instance from a database cursor. + * + * @param cursor The [SqlCursor] containing attachment data. + * @return An [Attachment] instance populated with data from the cursor. + */ public fun fromCursor(cursor: SqlCursor): Attachment = Attachment( id = cursor.getString(name = "id"), @@ -68,9 +89,9 @@ public data class Attachment( localUri = cursor.getStringOptional(name = "local_uri"), mediaType = cursor.getStringOptional(name = "media_type"), size = cursor.getLongOptional("size"), - state = cursor.getLong("state").toInt(), + state = AttachmentState.fromLong(cursor.getLong("state")), hasSynced = cursor.getLong("has_synced").toInt(), metaData = cursor.getStringOptional("meta_data"), ) } -} +} \ No newline at end of file diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 80c58069..f1a1f78a 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -323,18 +323,18 @@ public open class AttachmentQueue( Attachment( id = item.id, filename = filename, - state = AttachmentState.QUEUED_DOWNLOAD.ordinal, + state = AttachmentState.QUEUED_DOWNLOAD, metaData = item.metaData, ), ) } else if - (existingQueueItem.state == AttachmentState.ARCHIVED.ordinal) { + (existingQueueItem.state == AttachmentState.ARCHIVED) { // The attachment is present again. Need to queue it for sync. // We might be able to optimize this in future if (existingQueueItem.hasSynced == 1) { // No remote action required, we can restore the record (avoids deletion) attachmentUpdates.add( - existingQueueItem.copy(state = AttachmentState.SYNCED.ordinal), + existingQueueItem.copy(state = AttachmentState.SYNCED), ) } else { /** @@ -346,9 +346,9 @@ public open class AttachmentQueue( existingQueueItem.copy( state = if (existingQueueItem.localUri == null) { - AttachmentState.QUEUED_DOWNLOAD.ordinal + AttachmentState.QUEUED_DOWNLOAD } else { - AttachmentState.QUEUED_UPLOAD.ordinal + AttachmentState.QUEUED_UPLOAD }, ), ) @@ -361,10 +361,10 @@ public open class AttachmentQueue( */ currentAttachments .filter { - it.state != AttachmentState.QUEUED_DELETE.ordinal && + it.state != AttachmentState.QUEUED_DELETE && null == items.find { update -> update.id == it.id } }.forEach { - attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED.ordinal)) + attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED)) } attachmentsContext.saveAttachments(attachmentUpdates) @@ -412,7 +412,7 @@ public open class AttachmentQueue( filename = filename, size = fileSize, mediaType = mediaType, - state = AttachmentState.QUEUED_UPLOAD.ordinal, + state = AttachmentState.QUEUED_UPLOAD, localUri = localUri, metaData = metaData, ) @@ -451,7 +451,7 @@ public open class AttachmentQueue( db.writeTransaction { tx -> updateHook.invoke(tx, attachment) return@writeTransaction attachmentContext.upsertAttachment( - attachment.copy(state = AttachmentState.QUEUED_DELETE.ordinal), + attachment.copy(state = AttachmentState.QUEUED_DELETE), tx, ) } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index 36216fe1..0c176842 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -157,20 +157,23 @@ public class SyncingService( try { for (attachment in attachments) { when (attachment.state) { - AttachmentState.QUEUED_DOWNLOAD.ordinal -> { + AttachmentState.QUEUED_DOWNLOAD -> { logger.i("Downloading ${attachment.filename}") updatedAttachments.add(downloadAttachment(attachment)) } - AttachmentState.QUEUED_UPLOAD.ordinal -> { + AttachmentState.QUEUED_UPLOAD -> { logger.i("Uploading ${attachment.filename}") updatedAttachments.add(uploadAttachment(attachment)) } - AttachmentState.QUEUED_DELETE.ordinal -> { + AttachmentState.QUEUED_DELETE -> { logger.i("Deleting ${attachment.filename}") updatedAttachments.add(deleteAttachment(attachment)) } + + AttachmentState.SYNCED -> {} + AttachmentState.ARCHIVED -> {} } } @@ -199,14 +202,14 @@ public class SyncingService( attachment, ) logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") - return attachment.copy(state = AttachmentState.SYNCED.ordinal, hasSynced = 1) + return attachment.copy(state = AttachmentState.SYNCED, hasSynced = 1) } catch (e: Exception) { logger.e("Upload attachment error for attachment $attachment: ${e.message}") if (errorHandler != null) { val shouldRetry = errorHandler.onUploadError(attachment, e) if (!shouldRetry) { logger.i("Attachment with ID ${attachment.id} has been archived") - return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) + return attachment.copy(state = AttachmentState.ARCHIVED) } } @@ -234,7 +237,7 @@ public class SyncingService( // The attachment has been downloaded locally return attachment.copy( localUri = attachmentPath, - state = AttachmentState.SYNCED.ordinal, + state = AttachmentState.SYNCED, hasSynced = 1, ) } catch (e: Exception) { @@ -242,7 +245,7 @@ public class SyncingService( val shouldRetry = errorHandler.onDownloadError(attachment, e) if (!shouldRetry) { logger.i("Attachment with ID ${attachment.id} has been archived") - return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) + return attachment.copy(state = AttachmentState.ARCHIVED) } } @@ -261,13 +264,13 @@ public class SyncingService( if (attachment.localUri != null && localStorage.fileExists(attachment.localUri)) { localStorage.deleteFile(attachment.localUri) } - return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) + return attachment.copy(state = AttachmentState.ARCHIVED) } catch (e: Exception) { if (errorHandler != null) { val shouldRetry = errorHandler.onDeleteError(attachment, e) if (!shouldRetry) { logger.i("Attachment with ID ${attachment.id} has been archived") - return attachment.copy(state = AttachmentState.ARCHIVED.ordinal) + return attachment.copy(state = AttachmentState.ARCHIVED) } } // We'll retry this From 75f5b28965d5284ca0cec8daee8a541d8d3cf56e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 16:55:50 +0200 Subject: [PATCH 27/51] Sync Kotlin Gradle Plugin version --- gradle/libs.versions.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d20f8cfd..d1baf666 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,7 +12,6 @@ idea = "243.22562.218" # Meerkat | 2024.3.1 (see https://plugins.jetbrains.com/d kermit = "2.0.5" kotlin = "2.1.10" coroutines = "1.8.1" -kotlinGradlePlugin = "1.9.22" kotlinx-datetime = "0.6.2" kotlinx-io = "0.5.4" ktor = "3.0.1" @@ -60,7 +59,7 @@ configuration-annotations = { module = "co.touchlab.skie:configuration-annotatio gradle-download-task = { module = "de.undercouch:gradle-download-task", version.ref = "gradleDownloadTask" } kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } kermit-test = { module = "co.touchlab:kermit-test", version.ref = "kermit" } -kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlinGradlePlugin" } +kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } powersync-sqlite-core-android = { module = "co.powersync:powersync-sqlite-core", version.ref = "powersync-core" } mavenPublishPlugin = { module = "com.vanniktech:gradle-maven-publish-plugin", version.ref = "maven-publish" } From 5dba44aaab1e8249f3a70890796cae931c42a200 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 16:57:11 +0200 Subject: [PATCH 28/51] throw IllegalStateException when deleting a missing attachment file --- .../kotlin/com/powersync/attachments/AttachmentQueue.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index f1a1f78a..83f8b800 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -446,7 +446,7 @@ public open class AttachmentQueue( attachmentsService.withLock { attachmentContext -> val attachment = attachmentContext.getAttachment(attachmentId) - ?: throw Exception("Attachment record with id $attachmentId was not found.") + ?: throw error("Attachment record with id $attachmentId was not found.") db.writeTransaction { tx -> updateHook.invoke(tx, attachment) From f729d426c63a9bce1215b497f56ecc557a22f916 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 16:58:49 +0200 Subject: [PATCH 29/51] use fileSystem SystemFileSystem on IOLocalStorageAdapter --- .../storage/IOLocalStorageAdapter.kt | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt index c5d96c49..da71b00f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt @@ -12,6 +12,7 @@ import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.withContext import kotlinx.io.Buffer import kotlinx.io.buffered +import kotlinx.io.files.FileSystem import kotlinx.io.files.Path import kotlinx.io.files.SystemFileSystem @@ -19,6 +20,8 @@ import kotlinx.io.files.SystemFileSystem * Storage adapter for local storage using the KotlinX IO library */ public class IOLocalStorageAdapter : LocalStorage { + private val fileSystem: FileSystem = SystemFileSystem + public override suspend fun saveFile( filePath: String, data: Flow, @@ -26,7 +29,7 @@ public class IOLocalStorageAdapter : LocalStorage { runWrappedSuspending { withContext(Dispatchers.IO) { var totalSize = 0L - SystemFileSystem.sink(Path(filePath)).use { sink -> + fileSystem.sink(Path(filePath)).use { sink -> // Copy to a buffer in order to write Buffer().use { buffer -> data.collect { chunk -> @@ -48,7 +51,7 @@ public class IOLocalStorageAdapter : LocalStorage { mediaType: String?, ): Flow = flow { - SystemFileSystem.source(Path(filePath)).use { source -> + fileSystem.source(Path(filePath)).use { source -> source.buffered().use { bufferedSource -> var remaining = 0L val bufferSize = 8192L @@ -64,35 +67,35 @@ public class IOLocalStorageAdapter : LocalStorage { public override suspend fun deleteFile(filePath: String): Unit = runWrappedSuspending { withContext(Dispatchers.IO) { - SystemFileSystem.delete(Path(filePath)) + fileSystem.delete(Path(filePath)) } } public override suspend fun fileExists(filePath: String): Boolean = runWrappedSuspending { withContext(Dispatchers.IO) { - SystemFileSystem.exists(Path(filePath)) + fileSystem.exists(Path(filePath)) } } public override suspend fun makeDir(path: String): Unit = runWrappedSuspending { withContext(Dispatchers.IO) { - SystemFileSystem.createDirectories(Path(path)) + fileSystem.createDirectories(Path(path)) } } public override suspend fun rmDir(path: String): Unit = runWrappedSuspending { withContext(Dispatchers.IO) { - for (item in SystemFileSystem.list(Path(path))) { + for (item in fileSystem.list(Path(path))) { // Can't delete directories with files in them. Need to go down the file tree // and clear the directory. - val meta = SystemFileSystem.metadataOrNull(item) + val meta = fileSystem.metadataOrNull(item) if (meta?.isDirectory == true) { rmDir(item.toString()) } else if (meta?.isRegularFile == true) { - SystemFileSystem.delete(item) + fileSystem.delete(item) } } } @@ -104,8 +107,8 @@ public class IOLocalStorageAdapter : LocalStorage { ): Unit = runWrappedSuspending { withContext(Dispatchers.IO) { - SystemFileSystem.source(Path(sourcePath)).use { source -> - SystemFileSystem.sink(Path(targetPath)).use { sink -> + fileSystem.source(Path(sourcePath)).use { source -> + fileSystem.sink(Path(targetPath)).use { sink -> source.buffered().transferTo(sink.buffered()) } } From 66d1f9e9167bd5998f3bfcf2fc0dc0d23b89b38a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 17:02:47 +0200 Subject: [PATCH 30/51] use Duration for Flow throttle extension --- .../kotlin/com/powersync/attachments/sync/SyncingService.kt | 2 +- .../kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt | 3 ++- .../kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt | 5 +++-- .../commonMain/kotlin/com/powersync/utils/ThrottleFlow.kt | 5 +++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index 0c176842..ed220884 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -72,7 +72,7 @@ public class SyncingService( // while we are processing. We will always process on the trailing edge. // This buffer operation should automatically be applied to all merged sources. .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) - .throttle(syncThrottle.inWholeMilliseconds) + .throttle(syncThrottle) .collect { attachmentsService.withLock { context -> /** diff --git a/core/src/commonMain/kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt b/core/src/commonMain/kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt index bc9fe99c..c686bbd4 100644 --- a/core/src/commonMain/kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt @@ -44,6 +44,7 @@ import kotlinx.datetime.LocalDateTime import kotlinx.datetime.TimeZone import kotlinx.datetime.toInstant import kotlinx.serialization.encodeToString +import kotlin.time.Duration.Companion.milliseconds /** * A PowerSync managed database. @@ -231,7 +232,7 @@ internal class PowerSyncDatabaseImpl( internalDb .updatesOnTables() .filter { it.contains(InternalTable.CRUD.toString()) } - .throttle(crudThrottleMs) + .throttle(crudThrottleMs.milliseconds) .collect { stream.triggerCrudUploadAsync().join() } diff --git a/core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt b/core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt index ff0de31c..edf5130d 100644 --- a/core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt @@ -23,6 +23,7 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import kotlinx.serialization.encodeToString +import kotlin.time.Duration.Companion.milliseconds @OptIn(FlowPreview::class) internal class InternalDatabaseImpl( @@ -55,7 +56,7 @@ internal class InternalDatabaseImpl( private val dbContext = Dispatchers.IO companion object { - const val DEFAULT_WATCH_THROTTLE_MS = 30L + val DEFAULT_WATCH_THROTTLE = 30.milliseconds } override suspend fun execute( @@ -133,7 +134,7 @@ internal class InternalDatabaseImpl( // still trigger a trailing edge update. // Backpressure is avoided on the throttling and consumer level by buffering the last upstream value. // Note that the buffered upstream "value" only serves to trigger the getAll query. We don't buffer watch results. - .throttle(throttleMs ?: DEFAULT_WATCH_THROTTLE_MS) + .throttle(throttleMs?.milliseconds ?: DEFAULT_WATCH_THROTTLE) .collect { send(getAll(sql, parameters = parameters, mapper = mapper)) } diff --git a/core/src/commonMain/kotlin/com/powersync/utils/ThrottleFlow.kt b/core/src/commonMain/kotlin/com/powersync/utils/ThrottleFlow.kt index 711b2cb4..82d5c976 100644 --- a/core/src/commonMain/kotlin/com/powersync/utils/ThrottleFlow.kt +++ b/core/src/commonMain/kotlin/com/powersync/utils/ThrottleFlow.kt @@ -5,6 +5,7 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.buffer import kotlinx.coroutines.flow.flow +import kotlin.time.Duration /** * Throttles a flow with emissions on the leading and trailing edge. @@ -13,7 +14,7 @@ import kotlinx.coroutines.flow.flow * This throttle method acts as a slow consumer, but backpressure is not a concern * due to the conflated buffer dropping events during the throttle window. */ -internal fun Flow.throttle(windowMs: Long): Flow = +internal fun Flow.throttle(window: Duration): Flow = flow { // Use a buffer before throttle (ensure only the latest event is kept) val bufferedFlow = this@throttle.buffer(Channel.CONFLATED) @@ -23,7 +24,7 @@ internal fun Flow.throttle(windowMs: Long): Flow = emit(value) // Delay for the throttle window to avoid emitting too frequently - delay(windowMs) + delay(window) // The next incoming event will be provided from the buffer. // The next collect will emit the trailing edge From 35bbd4fe8fe44943c86308bad772e4d9a7e37e6c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 17:04:00 +0200 Subject: [PATCH 31/51] Fix periodic syncing loop --- .../kotlin/com/powersync/attachments/sync/SyncingService.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index ed220884..24396a4b 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -101,8 +101,10 @@ public class SyncingService( val periodicJob = launch { logger.i("Periodically syncing attachments") - syncTriggerFlow.emit(Unit) - delay(period) + while(true) { + syncTriggerFlow.emit(Unit) + delay(period) + } } watchJob.join() From e8fcf606053f2b03e1715789f865d8666849f3e4 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 17:04:37 +0200 Subject: [PATCH 32/51] revert test db path --- .../app/src/main/java/com/powersync/androidexample/App.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index a8b76577..bac02e25 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -47,7 +47,7 @@ fun App( ) } - val db = remember { PowerSyncDatabase(driverFactory, schema, dbFilename = "333.sqlite") } + val db = remember { PowerSyncDatabase(driverFactory, schema) } val attachments = remember { if (BuildConfig.SUPABASE_ATTACHMENT_BUCKET != "null") { From c48a2880522dabba04f6b0a83ad46db62c60b54a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 17:05:24 +0200 Subject: [PATCH 33/51] rename SUPABASE_STORAGE_BUCKET --- demos/android-supabase-todolist/app/build.gradle.kts | 4 ++-- .../app/src/main/java/com/powersync/androidexample/App.kt | 4 ++-- demos/android-supabase-todolist/local.properties.example | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/demos/android-supabase-todolist/app/build.gradle.kts b/demos/android-supabase-todolist/app/build.gradle.kts index 6157eeb3..a8011368 100644 --- a/demos/android-supabase-todolist/app/build.gradle.kts +++ b/demos/android-supabase-todolist/app/build.gradle.kts @@ -46,8 +46,8 @@ android { ) buildConfigField( "String", - "SUPABASE_ATTACHMENT_BUCKET", - "\"${getLocalProperty("SUPABASE_ATTACHMENT_BUCKET", "null")}\"", + "SUPABASE_STORAGE_BUCKET", + "\"${getLocalProperty("SUPABASE_STORAGE_BUCKET", "null")}\"", ) buildConfigField("String", "POWERSYNC_URL", "\"${getLocalProperty("POWERSYNC_URL", "")}\"") } diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index bac02e25..c1c8192c 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -43,14 +43,14 @@ fun App( powerSyncEndpoint = BuildConfig.POWERSYNC_URL, supabaseUrl = BuildConfig.SUPABASE_URL, supabaseKey = BuildConfig.SUPABASE_ANON_KEY, - storageBucket = BuildConfig.SUPABASE_ATTACHMENT_BUCKET, + storageBucket = BuildConfig.SUPABASE_STORAGE_BUCKET, ) } val db = remember { PowerSyncDatabase(driverFactory, schema) } val attachments = remember { - if (BuildConfig.SUPABASE_ATTACHMENT_BUCKET != "null") { + if (BuildConfig.SUPABASE_STORAGE_BUCKET != "null") { AttachmentQueue( db = db, remoteStorage = SupabaseRemoteStorage(supabase), diff --git a/demos/android-supabase-todolist/local.properties.example b/demos/android-supabase-todolist/local.properties.example index dd05ddce..189e1930 100644 --- a/demos/android-supabase-todolist/local.properties.example +++ b/demos/android-supabase-todolist/local.properties.example @@ -11,7 +11,7 @@ sdk.dir=/Users/dominic/Library/Android/sdk SUPABASE_URL=https://foo.supabase.co SUPABASE_ANON_KEY=foo -SUPABASE_ATTACHMENT_BUCKET=media # optional attachment bucket +SUPABASE_STORAGE_BUCKET=media # optional attachment bucket POWERSYNC_URL=https://foo.powersync.journeyapps.com # Set to true to use released PowerSync packages instead of the ones built locally. USE_RELEASED_POWERSYNC_VERSIONS=false From 31d7e5c66a7793097709671d5d76f751ff232ccf Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 17:20:17 +0200 Subject: [PATCH 34/51] fix build --- core/src/androidMain/kotlin/BuildConfig.kt | 2 +- .../app/src/main/java/com/powersync/androidexample/App.kt | 2 +- demos/android-supabase-todolist/local.properties.example | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/androidMain/kotlin/BuildConfig.kt b/core/src/androidMain/kotlin/BuildConfig.kt index 08f66eec..adfec803 100644 --- a/core/src/androidMain/kotlin/BuildConfig.kt +++ b/core/src/androidMain/kotlin/BuildConfig.kt @@ -2,4 +2,4 @@ public actual object BuildConfig { public actual val isDebug: Boolean get() = com.powersync.BuildConfig.DEBUG -} +} \ No newline at end of file diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index c1c8192c..af82b3b3 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -12,7 +12,6 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.rememberUpdatedState import androidx.compose.ui.Modifier import com.powersync.PowerSyncDatabase -import com.powersync.androidexample.BuildConfig import com.powersync.androidexample.ui.CameraService import com.powersync.attachments.AttachmentQueue import com.powersync.attachments.WatchedAttachmentItem @@ -30,6 +29,7 @@ import com.powersync.demos.screens.SignInScreen import com.powersync.demos.screens.SignUpScreen import com.powersync.demos.screens.TodosScreen import kotlinx.coroutines.runBlocking +import com.powersync.androidexample.BuildConfig @Composable fun App( diff --git a/demos/android-supabase-todolist/local.properties.example b/demos/android-supabase-todolist/local.properties.example index 189e1930..a1ac1145 100644 --- a/demos/android-supabase-todolist/local.properties.example +++ b/demos/android-supabase-todolist/local.properties.example @@ -11,7 +11,7 @@ sdk.dir=/Users/dominic/Library/Android/sdk SUPABASE_URL=https://foo.supabase.co SUPABASE_ANON_KEY=foo -SUPABASE_STORAGE_BUCKET=media # optional attachment bucket +SUPABASE_STORAGE_BUCKET=media # optional attachment bucket, set to null to disable POWERSYNC_URL=https://foo.powersync.journeyapps.com # Set to true to use released PowerSync packages instead of the ones built locally. USE_RELEASED_POWERSYNC_VERSIONS=false From 89ea92c3217eea12bc56a8a2b544bb13e6139eee Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 17:37:56 +0200 Subject: [PATCH 35/51] update shared build plugin --- gradle/libs.versions.toml | 3 +- .../src/main/kotlin/SharedBuildPlugin.kt | 46 ++++++++++++++----- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 34ac97ef..336a60e3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -62,7 +62,6 @@ kermit-test = { module = "co.touchlab:kermit-test", version.ref = "kermit" } kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } powersync-sqlite-core-android = { module = "co.powersync:powersync-sqlite-core", version.ref = "powersync-core" } mavenPublishPlugin = { module = "com.vanniktech:gradle-maven-publish-plugin", version.ref = "maven-publish" } -kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } test-junit = { group = "junit", name = "junit", version.ref = "junit" } test-android-runner = { module = "androidx.test:runner", version.ref = "androidx-test-runner" } @@ -102,7 +101,7 @@ sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } stately-concurrency = { module = "co.touchlab:stately-concurrency", version.ref = "stately" } supabase-client = { module = "io.github.jan-tennert.supabase:postgrest-kt", version.ref = "supabase" } supabase-auth = { module = "io.github.jan-tennert.supabase:auth-kt", version.ref = "supabase" } -supabase-storage = {module = "io.github.jan-tennert.supabase:storage-kt", version.ref = "supabase"} +supabase-storage = { module = "io.github.jan-tennert.supabase:storage-kt", version.ref = "supabase" } androidx-sqliteFramework = { module = "androidx.sqlite:sqlite-framework", version.ref = "androidxSqlite" } # Sample - Android diff --git a/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt b/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt index 302740c7..04ad5123 100644 --- a/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt +++ b/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt @@ -4,11 +4,12 @@ import de.undercouch.gradle.tasks.download.Download import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.artifacts.VersionCatalogsExtension -import org.gradle.api.tasks.Copy +import org.gradle.api.tasks.Exec import org.gradle.kotlin.dsl.withType import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget import org.jetbrains.kotlin.konan.target.Family +import java.io.File class SharedBuildPlugin : Plugin { override fun apply(project: Project) { @@ -32,16 +33,24 @@ class SharedBuildPlugin : Plugin { onlyIfModified(true) } - project.tasks.register("unzipPowersyncFramework", Copy::class.java) { - dependsOn(downloadPowersyncFramework) + val unzipPowersyncFramework = + project.tasks.register("unzipPowersyncFramework", Exec::class.java) { + dependsOn(downloadPowersyncFramework) - from( - project.zipTree(downloadPowersyncFramework.get().dest).matching { - include("powersync-sqlite-core.xcframework/**") - }, - ) - into(binariesFolder.map { it.dir("framework") }) - } + val zipfile = downloadPowersyncFramework.get().dest + inputs.file(zipfile) + val destination = File(zipfile.parentFile, "extracted") + doFirst { + destination.deleteRecursively() + destination.mkdir() + } + + // We're using unzip here because the Gradle copy task doesn't support symlinks. + executable = "unzip" + args(zipfile.absolutePath) + workingDir(destination) + outputs.dir(destination) + } project.extensions .getByType(KotlinMultiplatformExtension::class.java) @@ -56,7 +65,7 @@ class SharedBuildPlugin : Plugin { binaries .withType() .configureEach { - linkTaskProvider.configure { dependsOn("unzipPowersyncFramework") } + linkTaskProvider.configure { dependsOn(unzipPowersyncFramework) } linkerOpts("-framework", "powersync-sqlite-core") val frameworkRoot = @@ -65,6 +74,21 @@ class SharedBuildPlugin : Plugin { .get() .asFile.path + linkerOpts("-F", frameworkRoot) + linkerOpts("-rpath", frameworkRoot) + } + } else if (konanTarget.family == Family.OSX) { + binaries + .withType() + .configureEach { + linkTaskProvider.configure { dependsOn("unzipPowersyncFramework") } + linkerOpts("-framework", "powersync-sqlite-core") + var frameworkRoot = + binariesFolder + .map { it.dir("framework/extracted/powersync-sqlite-core.xcframework/macos-arm64_x86_64") } + .get() + .asFile.path + linkerOpts("-F", frameworkRoot) linkerOpts("-rpath", frameworkRoot) } From b0723b1a246b496232bf21ea7d29cb5110a8858b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 16 Apr 2025 18:24:48 +0200 Subject: [PATCH 36/51] wip: improve coroutine scope concurrency --- .../kotlin/com/powersync/AttachmentsTest.kt | 26 +- .../powersync/attachments/AttachmentQueue.kt | 749 +++++++++--------- .../implementation/AttachmentContextImpl.kt | 2 +- .../attachments/sync/SyncingService.kt | 378 +++++---- .../kotlin/com/powersync/utils/JsonTest.kt | 3 +- 5 files changed, 593 insertions(+), 565 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 1e937a8c..faa1379f 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -132,12 +132,12 @@ class AttachmentsTest { * We should assert that the download happens eventually. */ - if (attachmentRecord.state == AttachmentState.QUEUED_DOWNLOAD.ordinal) { + if (attachmentRecord.state == AttachmentState.QUEUED_DOWNLOAD) { // Wait for the download to be triggered attachmentRecord = attachmentQuery.awaitItem().first() } - attachmentRecord.state shouldBe AttachmentState.SYNCED.ordinal + attachmentRecord.state shouldBe AttachmentState.SYNCED // A download should have been attempted for this file verifySuspend { remote.downloadFile(attachmentMatcher(attachmentRecord)) } @@ -164,7 +164,7 @@ class AttachmentsTest { * The file should be deleted eventually */ var nextRecord: Attachment? = attachmentQuery.awaitItem().first() - if (nextRecord?.state == AttachmentState.ARCHIVED.ordinal) { + if (nextRecord?.state == AttachmentState.ARCHIVED) { nextRecord = attachmentQuery.awaitItem().getOrNull(0) } @@ -245,12 +245,12 @@ class AttachmentsTest { var attachmentRecord = attachmentQuery.awaitItem().first() attachmentRecord shouldNotBe null - if (attachmentRecord.state == AttachmentState.QUEUED_UPLOAD.ordinal) { + if (attachmentRecord.state == AttachmentState.QUEUED_UPLOAD) { // Wait for it to be synced attachmentRecord = attachmentQuery.awaitItem().first() } - attachmentRecord.state shouldBe AttachmentState.SYNCED.ordinal + attachmentRecord.state shouldBe AttachmentState.SYNCED // A download should have been attempted for this file verifySuspend { @@ -276,7 +276,7 @@ class AttachmentsTest { ) var nextRecord: Attachment? = attachmentQuery.awaitItem().first() - if (nextRecord?.state == AttachmentState.ARCHIVED.ordinal) { + if (nextRecord?.state == AttachmentState.ARCHIVED) { nextRecord = attachmentQuery.awaitItem().getOrNull(0) } @@ -351,7 +351,7 @@ class AttachmentsTest { waitFor { val record = attachmentQuery.awaitItem().first() record shouldNotBe null - record.state shouldBe AttachmentState.SYNCED.ordinal + record.state shouldBe AttachmentState.SYNCED } queue.deleteFile( @@ -452,12 +452,12 @@ class AttachmentsTest { * We should assert that the download happens eventually. */ - if (attachmentRecord.state == AttachmentState.QUEUED_DOWNLOAD.ordinal) { + if (attachmentRecord.state == AttachmentState.QUEUED_DOWNLOAD) { // Wait for the download to be triggered attachmentRecord = attachmentQuery.awaitItem().first() } - attachmentRecord.state shouldBe AttachmentState.SYNCED.ordinal + attachmentRecord.state shouldBe AttachmentState.SYNCED // A download should have been attempted for this file verifySuspend { remote.downloadFile(attachmentMatcher(attachmentRecord)) } @@ -478,7 +478,7 @@ class AttachmentsTest { ) attachmentRecord = attachmentQuery.awaitItem().first() - attachmentRecord.state shouldBe AttachmentState.ARCHIVED.ordinal + attachmentRecord.state shouldBe AttachmentState.ARCHIVED // Now if we set the photo_id, the archived record should be restored database.execute( @@ -493,7 +493,7 @@ class AttachmentsTest { ) attachmentRecord = attachmentQuery.awaitItem().first() - attachmentRecord.state shouldBe AttachmentState.SYNCED.ordinal + attachmentRecord.state shouldBe AttachmentState.SYNCED attachmentQuery.cancel() } @@ -570,12 +570,12 @@ class AttachmentsTest { var attachmentRecord = attachmentQuery.awaitItem().first() attachmentRecord shouldNotBe null - attachmentRecord.state shouldBe AttachmentState.QUEUED_DOWNLOAD.ordinal + attachmentRecord.state shouldBe AttachmentState.QUEUED_DOWNLOAD // The download should fail. We don't specify a retry. The record should be archived. attachmentRecord = attachmentQuery.awaitItem().first() - attachmentRecord.state shouldBe AttachmentState.ARCHIVED.ordinal + attachmentRecord.state shouldBe AttachmentState.ARCHIVED attachmentQuery.cancel() } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 83f8b800..a19e0974 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -9,9 +9,12 @@ import com.powersync.attachments.sync.SyncingService import com.powersync.db.internal.ConnectionContext import com.powersync.db.runWrappedSuspending import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex @@ -56,150 +59,152 @@ public data class WatchedAttachmentItem( * AbstractRemoteStorageAdapter and an attachment directory name which will * determine which folder attachments are stored into. */ -public open class AttachmentQueue( - /** - * PowerSync database client - */ - public val db: PowerSyncDatabase, - /** - * Adapter which interfaces with the remote storage backend - */ - public val remoteStorage: RemoteStorage, - /** - * Directory name where attachment files will be written to disk. - * This will be created if it does not exist - */ - private val attachmentsDirectory: String, - /** - * A flow for the current state of local attachments - * ```kotlin - * watchedAttachment = db.watch( - * sql = - * """ - * SELECT - * photo_id as id - * FROM - * checklists - * WHERE - * photo_id IS NOT NULL - * """, - * ) { cursor -> - * WatchedAttachmentItem( - * id = cursor.getString("id"), - * fileExtension = "jpg", - * ) - * } - * ``` - */ - private val watchedAttachments: Flow>, - /** - * Provides access to local filesystem storage methods - */ - public val localStorage: LocalStorage = IOLocalStorageAdapter(), - /** - * SQLite table where attachment state will be recorded - */ - private val attachmentsQueueTableName: String = DEFAULT_TABLE_NAME, - /** - * Attachment operation error handler. This specified if failed attachment operations - * should be retried. - */ - private val errorHandler: SyncErrorHandler? = null, - /** - * Periodic interval to trigger attachment sync operations - */ - private val syncInterval: Duration = 30.seconds, - /** - * Archived attachments can be used as a cache which can be restored if an attachment id - * reappears after being removed. This parameter defines how many archived records are retained. - * Records are deleted once the number of items exceeds this value. - */ - private val archivedCacheLimit: Long = 100, - /** - * Throttles remote sync operations triggering - */ - private val syncThrottleDuration: Duration = 1.seconds, - /** - * Creates a list of subdirectories in the {attachmentDirectoryName} directory - */ - private val subdirectories: List? = null, - /** - * Should attachments be downloaded - */ - private val downloadAttachments: Boolean = true, - /** - * Logging interface used for all log operations - */ - public val logger: Logger = Logger, -) { - public companion object { - public const val DEFAULT_TABLE_NAME: String = "attachments" - public const val DEFAULT_ATTACHMENTS_DIRECTORY_NAME: String = "attachments" - } - - /** - * Service which provides access to attachment records. - * Use this to: - * - Query all current attachment records - * - Create new attachment records for upload/download - */ - public val attachmentsService: AttachmentService = - AttachmentServiceImpl( - db, - attachmentsQueueTableName, - logger, - maxArchivedCount = archivedCacheLimit, - ) - - /** - * Syncing service for this attachment queue. - * This processes attachment records and performs relevant upload, download and delete - * operations. - */ - private val syncingService: SyncingService = - SyncingService( - remoteStorage, - localStorage, - attachmentsService, - ::getLocalUri, - errorHandler, - logger, - syncThrottleDuration, - ) - - private var syncStatusJob: Job? = null - private val mutex = Mutex() - - public var closed: Boolean = false - - /** - * Initialize the attachment queue by - * 1. Creating attachments directory - * 2. Adding watches for uploads, downloads, and deletes - * 3. Adding trigger to run uploads, downloads, and deletes when device is online after being offline - */ - @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun startSync(): Unit = - runWrappedSuspending { - mutex.withLock { - if (closed) { - throw Exception("Attachment queue has been closed") - } - // Ensure the directory where attachments are downloaded, exists - localStorage.makeDir(attachmentsDirectory) +public open class AttachmentQueue + @OptIn(DelicateCoroutinesApi::class) + constructor( + /** + * PowerSync database client + */ + public val db: PowerSyncDatabase, + /** + * Adapter which interfaces with the remote storage backend + */ + public val remoteStorage: RemoteStorage, + /** + * Directory name where attachment files will be written to disk. + * This will be created if it does not exist + */ + private val attachmentsDirectory: String, + /** + * A flow for the current state of local attachments + * ```kotlin + * watchedAttachment = db.watch( + * sql = + * """ + * SELECT + * photo_id as id + * FROM + * checklists + * WHERE + * photo_id IS NOT NULL + * """, + * ) { cursor -> + * WatchedAttachmentItem( + * id = cursor.getString("id"), + * fileExtension = "jpg", + * ) + * } + * ``` + */ + private val watchedAttachments: Flow>, + /** + * Provides access to local filesystem storage methods + */ + public val localStorage: LocalStorage = IOLocalStorageAdapter(), + /** + * SQLite table where attachment state will be recorded + */ + private val attachmentsQueueTableName: String = DEFAULT_TABLE_NAME, + /** + * Attachment operation error handler. This specified if failed attachment operations + * should be retried. + */ + private val errorHandler: SyncErrorHandler? = null, + /** + * Periodic interval to trigger attachment sync operations + */ + private val syncInterval: Duration = 30.seconds, + /** + * Archived attachments can be used as a cache which can be restored if an attachment id + * reappears after being removed. This parameter defines how many archived records are retained. + * Records are deleted once the number of items exceeds this value. + */ + private val archivedCacheLimit: Long = 100, + /** + * Throttles remote sync operations triggering + */ + private val syncThrottleDuration: Duration = 1.seconds, + /** + * Creates a list of subdirectories in the {attachmentDirectoryName} directory + */ + private val subdirectories: List? = null, + /** + * Should attachments be downloaded + */ + private val downloadAttachments: Boolean = true, + /** + * Logging interface used for all log operations + */ + public val logger: Logger = Logger, + ) { + public companion object { + public const val DEFAULT_TABLE_NAME: String = "attachments" + public const val DEFAULT_ATTACHMENTS_DIRECTORY_NAME: String = "attachments" + } - subdirectories?.forEach { subdirectory -> - localStorage.makeDir(Path(attachmentsDirectory, subdirectory).toString()) - } + /** + * Service which provides access to attachment records. + * Use this to: + * - Query all current attachment records + * - Create new attachment records for upload/download + */ + public val attachmentsService: AttachmentService = + AttachmentServiceImpl( + db, + attachmentsQueueTableName, + logger, + maxArchivedCount = archivedCacheLimit, + ) + + private val supervisorJob = SupervisorJob() + private val syncScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private var syncStatusJob: Job? = null + private val mutex = Mutex() + + /** + * Syncing service for this attachment queue. + * This processes attachment records and performs relevant upload, download and delete + * operations. + */ + private val syncingService = + SyncingService( + remoteStorage, + localStorage, + attachmentsService, + ::getLocalUri, + errorHandler, + logger, + syncScope, + syncThrottleDuration, + ) + + public var closed: Boolean = false + + /** + * Initialize the attachment queue by + * 1. Creating attachments directory + * 2. Adding watches for uploads, downloads, and deletes + * 3. Adding trigger to run uploads, downloads, and deletes when device is online after being offline + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun startSync(): Unit = + runWrappedSuspending { + mutex.withLock { + if (closed) { + throw Exception("Attachment queue has been closed") + } + // Ensure the directory where attachments are downloaded, exists + localStorage.makeDir(attachmentsDirectory) - val scope = CoroutineScope(Dispatchers.IO) + subdirectories?.forEach { subdirectory -> + localStorage.makeDir(Path(attachmentsDirectory, subdirectory).toString()) + } - syncingService.startSync(syncInterval) + syncingService.startSync(syncInterval) - // Listen for connectivity changes - syncStatusJob = - scope.launch { - val statusJob = + // Listen for connectivity changes + syncStatusJob = + syncScope.launch { launch { var previousConnected = db.currentStatus.connected db.currentStatus.asFlow().collect { status -> @@ -210,280 +215,274 @@ public open class AttachmentQueue( } } - val watchJob = launch { // Watch local attachment relationships and sync the attachment records watchedAttachments.collect { items -> processWatchedAttachments(items) } } + } + } + } - statusJob.join() - watchJob.join() + /** + * Stops syncing. Syncing may be resumed with [startSync]. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun stopSyncing(): Unit = + runWrappedSuspending { + mutex.withLock { + if (closed) { + return@runWrappedSuspending } - } - } - /** - * Stops syncing. Syncing may be resumed with [startSync]. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun stopSyncing(): Unit = - runWrappedSuspending { - mutex.withLock { - if (closed) { - return@runWrappedSuspending + syncStatusJob?.cancelAndJoin() + syncingService.stopSync() } - - syncStatusJob?.cancel() - syncStatusJob?.join() - - syncingService.stopSync() } - } - /** - * Closes the queue. - * The queue cannot be used after closing. - * A new queue should be created. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun close(): Unit = - runWrappedSuspending { - mutex.withLock { - if (closed) { - return@runWrappedSuspending - } + /** + * Closes the queue. + * The queue cannot be used after closing. + * A new queue should be created. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun close(): Unit = + runWrappedSuspending { + mutex.withLock { + if (closed) { + return@runWrappedSuspending + } - syncStatusJob?.cancel() - syncStatusJob?.join() - syncingService.close() - closed = true + syncStatusJob?.cancelAndJoin() + syncingService.close() + supervisorJob.cancelAndJoin() + closed = true + } } - } - - /** - * Resolves the filename for new attachment items. - * A new attachment from [watchAttachments] might not include a filename. - * Concatenates the attachment ID an extension by default. - * This method can be overriden for custom behaviour. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun resolveNewAttachmentFilename( - attachmentId: String, - fileExtension: String?, - ): String = "$attachmentId.$fileExtension" - /** - * Processes attachment items returned from [watchAttachments]. - * The default implementation assets the items returned from [watchAttachments] as the definitive - * state for local attachments. - * - * Records currently in the attachment queue which are not present in the items are deleted from - * the queue. - * - * Received items which are not currently in the attachment queue are assumed scheduled for - * download. This requires that locally created attachments should be created with [saveFile] - * before assigning the attachment ID to the relevant watched tables. - * - * This method can be overriden for custom behaviour. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun processWatchedAttachments(items: List): Unit = - runWrappedSuspending { - /** - * Use a lock here to prevent conflicting state updates - */ - attachmentsService.withLock { attachmentsContext -> + /** + * Resolves the filename for new attachment items. + * A new attachment from [watchAttachments] might not include a filename. + * Concatenates the attachment ID an extension by default. + * This method can be overriden for custom behaviour. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public open suspend fun resolveNewAttachmentFilename( + attachmentId: String, + fileExtension: String?, + ): String = "$attachmentId.$fileExtension" + + /** + * Processes attachment items returned from [watchAttachments]. + * The default implementation assets the items returned from [watchAttachments] as the definitive + * state for local attachments. + * + * Records currently in the attachment queue which are not present in the items are deleted from + * the queue. + * + * Received items which are not currently in the attachment queue are assumed scheduled for + * download. This requires that locally created attachments should be created with [saveFile] + * before assigning the attachment ID to the relevant watched tables. + * + * This method can be overriden for custom behaviour. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public open suspend fun processWatchedAttachments(items: List): Unit = + runWrappedSuspending { /** - * Need to get all the attachments which are tracked in the DB. - * We might need to restore an archived attachment. + * Use a lock here to prevent conflicting state updates */ - val currentAttachments = attachmentsContext.getAttachments() - val attachmentUpdates = mutableListOf() + attachmentsService.withLock { attachmentsContext -> + /** + * Need to get all the attachments which are tracked in the DB. + * We might need to restore an archived attachment. + */ + val currentAttachments = attachmentsContext.getAttachments() + val attachmentUpdates = mutableListOf() - for (item in items) { - val existingQueueItem = currentAttachments.find { it.id == item.id } + for (item in items) { + val existingQueueItem = currentAttachments.find { it.id == item.id } - if (existingQueueItem == null) { - if (!downloadAttachments) { - continue - } - // This item should be added to the queue - // This item is assumed to be coming from an upstream sync - // Locally created new items should be persisted using [saveFile] before - // this point. - val filename = - item.filename ?: resolveNewAttachmentFilename( - attachmentId = item.id, - fileExtension = item.fileExtension, - ) + if (existingQueueItem == null) { + if (!downloadAttachments) { + continue + } + // This item should be added to the queue + // This item is assumed to be coming from an upstream sync + // Locally created new items should be persisted using [saveFile] before + // this point. + val filename = + item.filename ?: resolveNewAttachmentFilename( + attachmentId = item.id, + fileExtension = item.fileExtension, + ) - attachmentUpdates.add( - Attachment( - id = item.id, - filename = filename, - state = AttachmentState.QUEUED_DOWNLOAD, - metaData = item.metaData, - ), - ) - } else if - (existingQueueItem.state == AttachmentState.ARCHIVED) { - // The attachment is present again. Need to queue it for sync. - // We might be able to optimize this in future - if (existingQueueItem.hasSynced == 1) { - // No remote action required, we can restore the record (avoids deletion) attachmentUpdates.add( - existingQueueItem.copy(state = AttachmentState.SYNCED), - ) - } else { - /** - * The localURI should be set if the record was meant to be downloaded - * and has been synced. If it's missing and hasSynced is false then - * it must be an upload operation - */ - attachmentUpdates.add( - existingQueueItem.copy( - state = - if (existingQueueItem.localUri == null) { - AttachmentState.QUEUED_DOWNLOAD - } else { - AttachmentState.QUEUED_UPLOAD - }, + Attachment( + id = item.id, + filename = filename, + state = AttachmentState.QUEUED_DOWNLOAD, + metaData = item.metaData, ), ) + } else if + (existingQueueItem.state == AttachmentState.ARCHIVED) { + // The attachment is present again. Need to queue it for sync. + // We might be able to optimize this in future + if (existingQueueItem.hasSynced == 1) { + // No remote action required, we can restore the record (avoids deletion) + attachmentUpdates.add( + existingQueueItem.copy(state = AttachmentState.SYNCED), + ) + } else { + /** + * The localURI should be set if the record was meant to be downloaded + * and has been synced. If it's missing and hasSynced is false then + * it must be an upload operation + */ + attachmentUpdates.add( + existingQueueItem.copy( + state = + if (existingQueueItem.localUri == null) { + AttachmentState.QUEUED_DOWNLOAD + } else { + AttachmentState.QUEUED_UPLOAD + }, + ), + ) + } } } + + /** + * Archive any items not specified in the watched items except for items pending delete. + */ + currentAttachments + .filter { + it.state != AttachmentState.QUEUED_DELETE && + null == items.find { update -> update.id == it.id } + }.forEach { + attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED)) + } + + attachmentsContext.saveAttachments(attachmentUpdates) } + } + + /** + * A function which creates a new attachment locally. This new attachment is queued for upload + * after creation. + * + * The filename is resolved using [resolveNewAttachmentFilename]. + * + * A [updateHook] is provided which should be used when assigning relationships to the newly + * created attachment. This hook is executed in the same writeTransaction which creates the + * attachment record. + * + * This method can be overriden for custom behaviour. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public open suspend fun saveFile( + data: Flow, + mediaType: String, + fileExtension: String? = null, + metaData: String? = null, + updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, + ): Attachment = + runWrappedSuspending { + val id = db.get("SELECT uuid()") { it.getString(0)!! } + val filename = + resolveNewAttachmentFilename(attachmentId = id, fileExtension = fileExtension) + val localUri = getLocalUri(filename) + + // write the file to the filesystem + val fileSize = localStorage.saveFile(localUri, data) /** - * Archive any items not specified in the watched items except for items pending delete. + * Starts a write transaction. The attachment record and relevant local relationship + * assignment should happen in the same transaction. */ - currentAttachments - .filter { - it.state != AttachmentState.QUEUED_DELETE && - null == items.find { update -> update.id == it.id } - }.forEach { - attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED)) - } + attachmentsService.withLock { attachmentContext -> + db.writeTransaction { tx -> + val attachment = + Attachment( + id = id, + filename = filename, + size = fileSize, + mediaType = mediaType, + state = AttachmentState.QUEUED_UPLOAD, + localUri = localUri, + metaData = metaData, + ) - attachmentsContext.saveAttachments(attachmentUpdates) - } - } + /** + * Allow consumers to set relationships to this attachment id + */ + updateHook.invoke(tx, attachment) - /** - * A function which creates a new attachment locally. This new attachment is queued for upload - * after creation. - * - * The filename is resolved using [resolveNewAttachmentFilename]. - * - * A [updateHook] is provided which should be used when assigning relationships to the newly - * created attachment. This hook is executed in the same writeTransaction which creates the - * attachment record. - * - * This method can be overriden for custom behaviour. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun saveFile( - data: Flow, - mediaType: String, - fileExtension: String? = null, - metaData: String? = null, - updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, - ): Attachment = - runWrappedSuspending { - val id = db.get("SELECT uuid()") { it.getString(0)!! } - val filename = - resolveNewAttachmentFilename(attachmentId = id, fileExtension = fileExtension) - val localUri = getLocalUri(filename) - - // write the file to the filesystem - val fileSize = localStorage.saveFile(localUri, data) - - /** - * Starts a write transaction. The attachment record and relevant local relationship - * assignment should happen in the same transaction. - */ - attachmentsService.withLock { attachmentContext -> - db.writeTransaction { tx -> - val attachment = - Attachment( - id = id, - filename = filename, - size = fileSize, - mediaType = mediaType, - state = AttachmentState.QUEUED_UPLOAD, - localUri = localUri, - metaData = metaData, + return@writeTransaction attachmentContext.upsertAttachment( + attachment, + tx, ) - - /** - * Allow consumers to set relationships to this attachment id - */ - updateHook.invoke(tx, attachment) - - return@writeTransaction attachmentContext.upsertAttachment( - attachment, - tx, - ) + } } } - } - /** - * A function which creates an attachment delete operation locally. This operation is queued - * for delete. - * The default implementation assumes the attachment record already exists locally. An exception - * is thrown if the record does not exist locally. - * This method can be overriden for custom behaviour. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun deleteFile( - attachmentId: String, - updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, - ): Attachment = - runWrappedSuspending { - attachmentsService.withLock { attachmentContext -> - val attachment = - attachmentContext.getAttachment(attachmentId) - ?: throw error("Attachment record with id $attachmentId was not found.") - - db.writeTransaction { tx -> - updateHook.invoke(tx, attachment) - return@writeTransaction attachmentContext.upsertAttachment( - attachment.copy(state = AttachmentState.QUEUED_DELETE), - tx, - ) + /** + * A function which creates an attachment delete operation locally. This operation is queued + * for delete. + * The default implementation assumes the attachment record already exists locally. An exception + * is thrown if the record does not exist locally. + * This method can be overriden for custom behaviour. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public open suspend fun deleteFile( + attachmentId: String, + updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, + ): Attachment = + runWrappedSuspending { + attachmentsService.withLock { attachmentContext -> + val attachment = + attachmentContext.getAttachment(attachmentId) + ?: throw error("Attachment record with id $attachmentId was not found.") + + db.writeTransaction { tx -> + updateHook.invoke(tx, attachment) + return@writeTransaction attachmentContext.upsertAttachment( + attachment.copy(state = AttachmentState.QUEUED_DELETE), + tx, + ) + } } } - } - /** - * Return users storage directory with the attachmentPath use to load the file. - * Example: filePath: "attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg" - */ - public open fun getLocalUri(filename: String): String = Path(attachmentsDirectory, filename).toString() - - /** - * Removes all archived items - */ - public suspend fun expireCache() { - var done: Boolean - attachmentsService.withLock { context -> - do { - done = syncingService.deleteArchivedAttachments(context) - } while (!done) + /** + * Return users storage directory with the attachmentPath use to load the file. + * Example: filePath: "attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg" + */ + public open fun getLocalUri(filename: String): String = Path(attachmentsDirectory, filename).toString() + + /** + * Removes all archived items + */ + public suspend fun expireCache() { + var done: Boolean + attachmentsService.withLock { context -> + do { + done = syncingService.deleteArchivedAttachments(context) + } while (!done) + } } - } - /** - * Clears the attachment queue and deletes all attachment files - */ - public suspend fun clearQueue() { - attachmentsService.withLock { - it.clearQueue() + /** + * Clears the attachment queue and deletes all attachment files + */ + public suspend fun clearQueue() { + attachmentsService.withLock { + it.clearQueue() + } + // Remove the attachments directory + localStorage.rmDir(attachmentsDirectory) } - // Remove the attachments directory - localStorage.rmDir(attachmentsDirectory) } -} diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt index 55d106a2..b48453f6 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt @@ -182,7 +182,7 @@ public open class AttachmentContextImpl( updatedRecord.localUri, updatedRecord.mediaType, updatedRecord.size, - updatedRecord.state, + updatedRecord.state.ordinal, updatedRecord.hasSynced, ), ) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index 24396a4b..de32f621 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -12,9 +12,9 @@ import com.powersync.attachments.SyncErrorHandler import com.powersync.utils.throttle import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.IO +import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Job +import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableSharedFlow @@ -28,37 +28,52 @@ import kotlin.time.Duration import kotlin.time.Duration.Companion.seconds /** - * Service used to sync attachments between local and remote storage + * Service responsible for syncing attachments between local and remote storage. + * + * This service handles downloading, uploading, and deleting attachments, as well as + * periodically syncing attachment states. It ensures proper lifecycle management + * of sync operations and provides mechanisms for error handling and retries. + * + * @property remoteStorage The remote storage implementation for handling file operations. + * @property localStorage The local storage implementation for managing files locally. + * @property attachmentsService The service for managing attachment states and operations. + * @property getLocalUri A function to resolve the local URI for a given filename. + * @property errorHandler Optional error handler for managing sync-related errors. + * @property logger Logger instance for logging sync operations and errors. + * @property syncThrottle The minimum duration between consecutive sync operations. + * @property scope The coroutine scope used for managing sync operations. */ -public class SyncingService( - private val remoteStorage: RemoteStorage, - private val localStorage: LocalStorage, - private val attachmentsService: AttachmentService, - private val getLocalUri: suspend (String) -> String, - private val errorHandler: SyncErrorHandler?, - private val logger: Logger, - private val syncThrottle: Duration = 5.seconds, -) { - private val scope = CoroutineScope(Dispatchers.IO) - private val mutex = Mutex() - private var syncJob: Job? = null +public class SyncingService + @OptIn(DelicateCoroutinesApi::class) + constructor( + private val remoteStorage: RemoteStorage, + private val localStorage: LocalStorage, + private val attachmentsService: AttachmentService, + private val getLocalUri: suspend (String) -> String, + private val errorHandler: SyncErrorHandler?, + private val logger: Logger, + private val scope: CoroutineScope, + private val syncThrottle: Duration = 5.seconds, + ) { + private val mutex = Mutex() + private var syncJob: Job? = null - /** - * Used to trigger the sync process either manually or periodically - */ - private val syncTriggerFlow = MutableSharedFlow(replay = 0) + /** + * Used to trigger the sync process either manually or periodically + */ + private val syncTriggerFlow = MutableSharedFlow(replay = 0) - /** - * Starts syncing operations - */ - public suspend fun startSync(period: Duration = 30.seconds): Unit = - mutex.withLock { - syncJob?.cancel() - syncJob?.join() + /** + * Starts the syncing process, including periodic and event-driven sync operations. + * + * @param period The interval at which periodic sync operations are triggered. + */ + public suspend fun startSync(period: Duration = 30.seconds): Unit = + mutex.withLock { + syncJob?.cancelAndJoin() - syncJob = - scope.launch { - val watchJob = + syncJob = + scope.launch { launch { merge( // Handles manual triggers for sync events @@ -98,186 +113,199 @@ public class SyncingService( } } - val periodicJob = launch { logger.i("Periodically syncing attachments") - while(true) { + while (true) { syncTriggerFlow.emit(Unit) delay(period) } } + } + } - watchJob.join() - periodicJob.join() - } + /** + * Enqueues a sync operation + */ + public suspend fun triggerSync() { + syncTriggerFlow.emit(Unit) } - /** - * Enqueues a sync operation - */ - public suspend fun triggerSync() { - syncTriggerFlow.emit(Unit) - } + /** + * Stops all ongoing sync operations. + */ + public suspend fun stopSync(): Unit = + mutex.withLock { + syncJob?.cancelAndJoin() + } - /** - * Stops syncing operations - */ - public suspend fun stopSync(): Unit = - mutex.withLock { - syncJob?.cancel() - syncJob?.join() + /** + * Closes the syncing service, stopping all operations and releasing resources. + */ + public suspend fun close() { + stopSync() } - /** - * Closes the syncing service. - */ - public suspend fun close() { - stopSync() - } - - public suspend fun deleteArchivedAttachments(context: AttachmentContext): Boolean = - context.deleteArchivedAttachments { pendingDelete -> - for (attachment in pendingDelete) { - if (attachment.localUri == null) { - continue - } - if (!localStorage.fileExists(attachment.localUri)) { - continue - } - localStorage.deleteFile(attachment.localUri) - } - } + /** + * Handles syncing operations for a list of attachments, including downloading, + * uploading, and deleting files based on their states. + * + * @param attachments The list of attachments to process. + * @param context The attachment context used for managing attachment states. + */ + private suspend fun handleSync( + attachments: List, + context: AttachmentContext, + ) { + val updatedAttachments = mutableListOf() + try { + for (attachment in attachments) { + when (attachment.state) { + AttachmentState.QUEUED_DOWNLOAD -> { + logger.i("Downloading ${attachment.filename}") + updatedAttachments.add(downloadAttachment(attachment)) + } - /** - * Handle downloading, uploading or deleting of attachments - */ - private suspend fun handleSync( - attachments: List, - context: AttachmentContext, - ) { - val updatedAttachments = mutableListOf() - try { - for (attachment in attachments) { - when (attachment.state) { - AttachmentState.QUEUED_DOWNLOAD -> { - logger.i("Downloading ${attachment.filename}") - updatedAttachments.add(downloadAttachment(attachment)) - } + AttachmentState.QUEUED_UPLOAD -> { + logger.i("Uploading ${attachment.filename}") + updatedAttachments.add(uploadAttachment(attachment)) + } - AttachmentState.QUEUED_UPLOAD -> { - logger.i("Uploading ${attachment.filename}") - updatedAttachments.add(uploadAttachment(attachment)) - } + AttachmentState.QUEUED_DELETE -> { + logger.i("Deleting ${attachment.filename}") + updatedAttachments.add(deleteAttachment(attachment)) + } - AttachmentState.QUEUED_DELETE -> { - logger.i("Deleting ${attachment.filename}") - updatedAttachments.add(deleteAttachment(attachment)) + AttachmentState.SYNCED -> {} + AttachmentState.ARCHIVED -> {} } - - AttachmentState.SYNCED -> {} - AttachmentState.ARCHIVED -> {} } - } - // Update the state of processed attachments - context.saveAttachments(updatedAttachments) - } catch (error: Exception) { - // We retry, on the next invocation, whenever there are errors on this level - logger.e("Error during sync: ${error.message}") + // Update the state of processed attachments + context.saveAttachments(updatedAttachments) + } catch (error: Exception) { + // We retry, on the next invocation, whenever there are errors on this level + logger.e("Error during sync: ${error.message}") + } } - } - /** - * Upload attachment from local storage to remote storage. - */ - private suspend fun uploadAttachment(attachment: Attachment): Attachment { - try { - if (attachment.localUri == null) { - throw PowerSyncException( - "No localUri for attachment $attachment", - cause = Exception("attachment.localUri == null"), - ) - } + /** + * Uploads an attachment from local storage to remote storage. + * + * @param attachment The attachment to upload. + * @return The updated attachment with its new state. + */ + private suspend fun uploadAttachment(attachment: Attachment): Attachment { + try { + if (attachment.localUri == null) { + throw PowerSyncException( + "No localUri for attachment $attachment", + cause = Exception("attachment.localUri == null"), + ) + } - remoteStorage.uploadFile( - localStorage.readFile(attachment.localUri), - attachment, - ) - logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") - return attachment.copy(state = AttachmentState.SYNCED, hasSynced = 1) - } catch (e: Exception) { - logger.e("Upload attachment error for attachment $attachment: ${e.message}") - if (errorHandler != null) { - val shouldRetry = errorHandler.onUploadError(attachment, e) - if (!shouldRetry) { - logger.i("Attachment with ID ${attachment.id} has been archived") - return attachment.copy(state = AttachmentState.ARCHIVED) + remoteStorage.uploadFile( + localStorage.readFile(attachment.localUri), + attachment, + ) + logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") + return attachment.copy(state = AttachmentState.SYNCED, hasSynced = 1) + } catch (e: Exception) { + logger.e("Upload attachment error for attachment $attachment: ${e.message}") + if (errorHandler != null) { + val shouldRetry = errorHandler.onUploadError(attachment, e) + if (!shouldRetry) { + logger.i("Attachment with ID ${attachment.id} has been archived") + return attachment.copy(state = AttachmentState.ARCHIVED) + } } - } - // Retry the upload (same state) - return attachment + // Retry the upload (same state) + return attachment + } } - } - /** - * Download attachment from remote storage and save it to local storage. - * Returns the updated state of the attachment. - */ - private suspend fun downloadAttachment(attachment: Attachment): Attachment { /** - * When downloading an attachment we take the filename and resolve - * the local_uri where the file will be stored + * Downloads an attachment from remote storage and saves it to local storage. + * + * @param attachment The attachment to download. + * @return The updated attachment with its new state. */ - val attachmentPath = getLocalUri(attachment.filename) + private suspend fun downloadAttachment(attachment: Attachment): Attachment { + /** + * When downloading an attachment we take the filename and resolve + * the local_uri where the file will be stored + */ + val attachmentPath = getLocalUri(attachment.filename) - try { - val fileFlow = remoteStorage.downloadFile(attachment) - localStorage.saveFile(attachmentPath, fileFlow) - logger.i("Downloaded file \"${attachment.id}\"") + try { + val fileFlow = remoteStorage.downloadFile(attachment) + localStorage.saveFile(attachmentPath, fileFlow) + logger.i("Downloaded file \"${attachment.id}\"") - // The attachment has been downloaded locally - return attachment.copy( - localUri = attachmentPath, - state = AttachmentState.SYNCED, - hasSynced = 1, - ) - } catch (e: Exception) { - if (errorHandler != null) { - val shouldRetry = errorHandler.onDownloadError(attachment, e) - if (!shouldRetry) { - logger.i("Attachment with ID ${attachment.id} has been archived") - return attachment.copy(state = AttachmentState.ARCHIVED) + // The attachment has been downloaded locally + return attachment.copy( + localUri = attachmentPath, + state = AttachmentState.SYNCED, + hasSynced = 1, + ) + } catch (e: Exception) { + if (errorHandler != null) { + val shouldRetry = errorHandler.onDownloadError(attachment, e) + if (!shouldRetry) { + logger.i("Attachment with ID ${attachment.id} has been archived") + return attachment.copy(state = AttachmentState.ARCHIVED) + } } - } - logger.e("Download attachment error for attachment $attachment: ${e.message}") - // Return the same state, this will cause a retry - return attachment + logger.e("Download attachment error for attachment $attachment: ${e.message}") + // Return the same state, this will cause a retry + return attachment + } } - } - /** - * Delete attachment from remote, local storage and then remove it from the queue. - */ - private suspend fun deleteAttachment(attachment: Attachment): Attachment { - try { - remoteStorage.deleteFile(attachment) - if (attachment.localUri != null && localStorage.fileExists(attachment.localUri)) { - localStorage.deleteFile(attachment.localUri) - } - return attachment.copy(state = AttachmentState.ARCHIVED) - } catch (e: Exception) { - if (errorHandler != null) { - val shouldRetry = errorHandler.onDeleteError(attachment, e) - if (!shouldRetry) { - logger.i("Attachment with ID ${attachment.id} has been archived") - return attachment.copy(state = AttachmentState.ARCHIVED) + /** + * Deletes an attachment from remote and local storage, and removes it from the queue. + * + * @param attachment The attachment to delete. + * @return The updated attachment with its new state. + */ + private suspend fun deleteAttachment(attachment: Attachment): Attachment { + try { + remoteStorage.deleteFile(attachment) + if (attachment.localUri != null && localStorage.fileExists(attachment.localUri)) { + localStorage.deleteFile(attachment.localUri) + } + return attachment.copy(state = AttachmentState.ARCHIVED) + } catch (e: Exception) { + if (errorHandler != null) { + val shouldRetry = errorHandler.onDeleteError(attachment, e) + if (!shouldRetry) { + logger.i("Attachment with ID ${attachment.id} has been archived") + return attachment.copy(state = AttachmentState.ARCHIVED) + } } + // We'll retry this + logger.e("Error deleting attachment: ${e.message}") + return attachment } - // We'll retry this - logger.e("Error deleting attachment: ${e.message}") - return attachment } + + /** + * Deletes archived attachments from local storage. + * + * @param context The attachment context used to retrieve and manage archived attachments. + * @return `true` if all archived attachments were successfully deleted, `false` otherwise. + */ + public suspend fun deleteArchivedAttachments(context: AttachmentContext): Boolean = + context.deleteArchivedAttachments { pendingDelete -> + for (attachment in pendingDelete) { + if (attachment.localUri == null) { + continue + } + if (!localStorage.fileExists(attachment.localUri)) { + continue + } + localStorage.deleteFile(attachment.localUri) + } + } } -} diff --git a/core/src/commonTest/kotlin/com/powersync/utils/JsonTest.kt b/core/src/commonTest/kotlin/com/powersync/utils/JsonTest.kt index aa2611e1..46182096 100644 --- a/core/src/commonTest/kotlin/com/powersync/utils/JsonTest.kt +++ b/core/src/commonTest/kotlin/com/powersync/utils/JsonTest.kt @@ -23,6 +23,7 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds class JsonTest { @Test @@ -53,7 +54,7 @@ class JsonTest { emit(3) delay(100) emit(4) - }.throttle(100) + }.throttle(100.milliseconds) .map { // Adding a delay here to simulate a slow consumer delay(1000) From 72c828840fd7d3d09e17b9f387cc229277eca525 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 09:20:56 +0200 Subject: [PATCH 37/51] accept input scope --- .../powersync/attachments/AttachmentQueue.kt | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index a19e0974..2235c423 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -136,6 +136,10 @@ public open class AttachmentQueue * Logging interface used for all log operations */ public val logger: Logger = Logger, + /** + * Optional scope to launch syncing jobs in. + */ + private val coroutineScope: CoroutineScope? = null, ) { public companion object { public const val DEFAULT_TABLE_NAME: String = "attachments" @@ -156,8 +160,8 @@ public open class AttachmentQueue maxArchivedCount = archivedCacheLimit, ) - private val supervisorJob = SupervisorJob() - private val syncScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val syncScope = coroutineScope ?: CoroutineScope(SupervisorJob() + Dispatchers.IO) + private var syncStatusJob: Job? = null private val mutex = Mutex() @@ -193,6 +197,9 @@ public open class AttachmentQueue if (closed) { throw Exception("Attachment queue has been closed") } + + stopSyncingInternal() + // Ensure the directory where attachments are downloaded, exists localStorage.makeDir(attachmentsDirectory) @@ -230,15 +237,18 @@ public open class AttachmentQueue */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun stopSyncing(): Unit = - runWrappedSuspending { - mutex.withLock { - if (closed) { - return@runWrappedSuspending - } + mutex.withLock { + stopSyncingInternal() + } - syncStatusJob?.cancelAndJoin() - syncingService.stopSync() + private suspend fun stopSyncingInternal(): Unit = + runWrappedSuspending { + if (closed) { + return@runWrappedSuspending } + + syncStatusJob?.cancelAndJoin() + syncingService.stopSync() } /** @@ -256,7 +266,10 @@ public open class AttachmentQueue syncStatusJob?.cancelAndJoin() syncingService.close() - supervisorJob.cancelAndJoin() + if (coroutineScope == null) { + // Only cancel the internal scope if we created it + syncScope.coroutineContext[Job]?.cancelAndJoin() + } closed = true } } From 93e3381f89b6f64a5c989ed32fd3fce591977d5b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 09:24:45 +0200 Subject: [PATCH 38/51] lint fixes --- .../connector/supabase/SupabaseRemoteStorage.kt | 6 +++--- core/src/androidMain/kotlin/BuildConfig.kt | 2 +- .../kotlin/com/powersync/attachments/Attachment.kt | 11 ++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseRemoteStorage.kt b/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseRemoteStorage.kt index 5c3661af..423f039d 100644 --- a/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseRemoteStorage.kt +++ b/connectors/supabase/src/commonMain/kotlin/com/powersync/connector/supabase/SupabaseRemoteStorage.kt @@ -13,7 +13,6 @@ import kotlinx.coroutines.flow.flowOf public class SupabaseRemoteStorage( public val connector: SupabaseConnector, ) : RemoteStorage { - /** * Uploads a file to the Supabase storage bucket. * @@ -25,7 +24,8 @@ public class SupabaseRemoteStorage( fileData: Flow, attachment: Attachment, ) { - val byteSize = attachment.size?.toInt() ?: error("Cannot upload a file with no byte size specified") + val byteSize = + attachment.size?.toInt() ?: error("Cannot upload a file with no byte size specified") // Supabase wants a single ByteArray val buffer = ByteArray(byteSize) var position = 0 @@ -54,4 +54,4 @@ public class SupabaseRemoteStorage( override suspend fun deleteFile(attachment: Attachment) { connector.storageBucket().delete(attachment.filename) } -} \ No newline at end of file +} diff --git a/core/src/androidMain/kotlin/BuildConfig.kt b/core/src/androidMain/kotlin/BuildConfig.kt index adfec803..08f66eec 100644 --- a/core/src/androidMain/kotlin/BuildConfig.kt +++ b/core/src/androidMain/kotlin/BuildConfig.kt @@ -2,4 +2,4 @@ public actual object BuildConfig { public actual val isDebug: Boolean get() = com.powersync.BuildConfig.DEBUG -} \ No newline at end of file +} diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt index 6d7c715f..377055e9 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt @@ -33,7 +33,9 @@ public enum class AttachmentState { /** * The attachment is archived and no longer actively synchronized. */ - ARCHIVED; + ARCHIVED, + + ; public companion object { /** @@ -43,10 +45,9 @@ public enum class AttachmentState { * @return The corresponding [AttachmentState]. * @throws IllegalArgumentException If the value does not match any [AttachmentState]. */ - public fun fromLong(value: Long): AttachmentState { - return entries.getOrNull(value.toInt()) + public fun fromLong(value: Long): AttachmentState = + entries.getOrNull(value.toInt()) ?: throw IllegalArgumentException("Invalid value for AttachmentState: $value") - } } } @@ -94,4 +95,4 @@ public data class Attachment( metaData = cursor.getStringOptional("meta_data"), ) } -} \ No newline at end of file +} From 6cdbb098c04f6aec1bba3de9e1c7eb2078cc5635 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 09:57:56 +0200 Subject: [PATCH 39/51] fix ios test build --- .../powersync/attachments/storage/IOLocalStorageAdapter.kt | 4 ++-- plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt index da71b00f..e194be2f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt @@ -2,7 +2,6 @@ package com.powersync.attachments.storage import com.powersync.attachments.LocalStorage import com.powersync.db.runWrappedSuspending -import io.ktor.utils.io.core.readBytes import io.ktor.utils.io.core.remaining import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO @@ -15,6 +14,7 @@ import kotlinx.io.buffered import kotlinx.io.files.FileSystem import kotlinx.io.files.Path import kotlinx.io.files.SystemFileSystem +import kotlinx.io.readByteArray /** * Storage adapter for local storage using the KotlinX IO library @@ -58,7 +58,7 @@ public class IOLocalStorageAdapter : LocalStorage { do { bufferedSource.request(bufferSize) remaining = bufferedSource.remaining - emit(bufferedSource.readBytes(remaining.toInt())) + emit(bufferedSource.readByteArray(remaining.toInt())) } while (remaining > 0) } } diff --git a/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt b/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt index 04ad5123..e9488485 100644 --- a/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt +++ b/plugins/build-plugin/src/main/kotlin/SharedBuildPlugin.kt @@ -70,7 +70,7 @@ class SharedBuildPlugin : Plugin { val frameworkRoot = binariesFolder - .map { it.dir("framework/powersync-sqlite-core.xcframework/ios-arm64_x86_64-simulator") } + .map { it.dir("framework/extracted/powersync-sqlite-core.xcframework/ios-arm64_x86_64-simulator") } .get() .asFile.path From 032e8b618783b017c7f18334b5b8174d7710db5e Mon Sep 17 00:00:00 2001 From: benitav Date: Thu, 17 Apr 2025 12:30:21 +0200 Subject: [PATCH 40/51] Readme polish --- .../kotlin/com/powersync/attachments/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index f1cc7094..197056f8 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -4,9 +4,7 @@ A [PowerSync](https://powersync.com) library to manage attachments (such as imag This package is included in the PowerSync Core module. -A concrete example implementation of these attachment helpers is available in the [Android Supabase Demo](/demos/android-supabase-todolist/README.md) app. - -## Alpha Release +### Alpha Release Attachment helpers are currently in an alpha state, intended strictly for testing. Expect breaking changes and instability as development continues. @@ -14,7 +12,7 @@ Do not rely on this package for production use. ## Usage -An `AttachmentQueue` is used to manage and sync attachments in your app. The attachments' state are +An `AttachmentQueue` is used to manage and sync attachments in your app. The attachments' state is stored in a local-only attachments table. ### Key Assumptions @@ -22,12 +20,13 @@ stored in a local-only attachments table. - Each attachment is identified by a unique ID - Attachments are immutable once created - Relational data should reference attachments using a foreign key column -- Relational data should reflect the holistic state of attachments at any given time. An existing - local attachment will deleted locally if no relational data references it. +- Relational data should reflect the holistic state of attachments at any given time. An existing local attachment will deleted locally if no relational data references it. ### Example Implementation -In this example, the user captures photos when checklist items are completed as part of an +See the [Android Supabase Demo](/demos/android-supabase-todolist/README.md) for a basic example of attachment syncing. + +In the example below, the user captures photos when checklist items are completed as part of an inspection workflow. 1. First, define your schema including the `checklist` table and the local-only attachments table: From ec011a58b70b87998aec7639667cda997c79dbbd Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 12:43:20 +0200 Subject: [PATCH 41/51] Cleanup KDoc comments. Sync with Swift implementation. --- .../kotlin/com/powersync/AttachmentsTest.kt | 11 +- .../powersync/attachments/AttachmentQueue.kt | 187 +++++++++++------- .../attachments/AttachmentService.kt | 2 +- .../powersync/attachments/AttachmentTable.kt | 5 +- .../com/powersync/attachments/LocalStorage.kt | 55 +++++- .../com/powersync/attachments/README.md | 24 +-- .../powersync/attachments/RemoteStorage.kt | 14 +- .../powersync/attachments/SyncErrorHandler.kt | 24 ++- .../implementation/AttachmentContextImpl.kt | 46 ++--- .../implementation/AttachmentServiceImpl.kt | 5 +- .../attachments/sync/SyncingService.kt | 2 +- .../java/com/powersync/androidexample/App.kt | 5 +- 12 files changed, 250 insertions(+), 130 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index faa1379f..2b958be3 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -87,7 +87,7 @@ class AttachmentsTest { db = database, remoteStorage = remote, attachmentsDirectory = getAttachmentsDir(), - watchedAttachments = watchAttachments(database), + watchAttachments = { watchAttachments(database) }, /** * Sets the cache limit to zero for this test. Archived records will * immediately be deleted. @@ -198,7 +198,7 @@ class AttachmentsTest { db = database, remoteStorage = remote, attachmentsDirectory = getAttachmentsDir(), - watchedAttachments = watchAttachments(database), + watchAttachments = { watchAttachments(database) }, /** * Sets the cache limit to zero for this test. Archived records will * immediately be deleted. @@ -260,7 +260,6 @@ class AttachmentsTest { ) } - // A file should now exist val localUri = attachmentRecord.localUri!! queue.localStorage.fileExists(localUri) shouldBe true @@ -309,7 +308,7 @@ class AttachmentsTest { db = database, remoteStorage = remote, attachmentsDirectory = getAttachmentsDir(), - watchedAttachments = watchAttachments(database), + watchAttachments = { watchAttachments(database) }, /** * Sets the cache limit to zero for this test. Archived records will * immediately be deleted. @@ -408,7 +407,7 @@ class AttachmentsTest { db = database, remoteStorage = remote, attachmentsDirectory = getAttachmentsDir(), - watchedAttachments = watchAttachments(database), + watchAttachments = { watchAttachments(database) }, /** * Keep some items in the cache */ @@ -521,7 +520,7 @@ class AttachmentsTest { db = database, remoteStorage = remote, attachmentsDirectory = getAttachmentsDir(), - watchedAttachments = watchAttachments(database), + watchAttachments = { watchAttachments(database) }, archivedCacheLimit = 0, errorHandler = object : SyncErrorHandler { diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 2235c423..3799357c 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -30,19 +30,19 @@ import kotlin.time.Duration.Companion.seconds */ public data class WatchedAttachmentItem( /** - * Id for the attachment record + * Id for the attachment record. */ public val id: String, /** - * File extension used to determine an internal filename for storage if no [filename] is provided + * File extension used to determine an internal filename for storage if no [filename] is provided. */ public val fileExtension: String? = null, /** - * Filename to store the attachment with + * Filename to store the attachment with. */ public val filename: String? = null, /** - * Optional meta data for the attachment record + * Optional metadata for the attachment record. */ public val metaData: String? = null, ) { @@ -54,86 +54,89 @@ public data class WatchedAttachmentItem( } /** - * Class used to implement the attachment queue + * Class used to implement the attachment queue. * Requires a PowerSyncDatabase, an implementation of - * AbstractRemoteStorageAdapter and an attachment directory name which will + * AbstractRemoteStorageAdapter, and an attachment directory name which will * determine which folder attachments are stored into. */ public open class AttachmentQueue @OptIn(DelicateCoroutinesApi::class) constructor( /** - * PowerSync database client + * PowerSync database client. */ public val db: PowerSyncDatabase, /** - * Adapter which interfaces with the remote storage backend + * Adapter which interfaces with the remote storage backend. */ public val remoteStorage: RemoteStorage, /** * Directory name where attachment files will be written to disk. - * This will be created if it does not exist + * This will be created if it does not exist. */ private val attachmentsDirectory: String, /** - * A flow for the current state of local attachments + * A flow generator for the current state of local attachments. + * Example: * ```kotlin - * watchedAttachment = db.watch( - * sql = - * """ - * SELECT - * photo_id as id - * FROM - * checklists - * WHERE - * photo_id IS NOT NULL - * """, - * ) { cursor -> - * WatchedAttachmentItem( - * id = cursor.getString("id"), - * fileExtension = "jpg", - * ) - * } + * watchAttachments = { + * db.watch( + * sql = """ + * SELECT + * photo_id as id, + * 'jpg' as fileExtension + * FROM + * checklists + * WHERE + * photo_id IS NOT NULL + * """, + * ) { cursor -> + * WatchedAttachmentItem( + * id = cursor.getString("id"), + * fileExtension = "jpg", + * ) + * } + * } * ``` */ - private val watchedAttachments: Flow>, + private val watchAttachments: () -> Flow>, /** - * Provides access to local filesystem storage methods + * Provides access to local filesystem storage methods. */ public val localStorage: LocalStorage = IOLocalStorageAdapter(), /** - * SQLite table where attachment state will be recorded + * SQLite table where attachment state will be recorded. */ private val attachmentsQueueTableName: String = DEFAULT_TABLE_NAME, /** - * Attachment operation error handler. This specified if failed attachment operations + * Attachment operation error handler. Specifies if failed attachment operations * should be retried. */ private val errorHandler: SyncErrorHandler? = null, /** - * Periodic interval to trigger attachment sync operations + * Periodic interval to trigger attachment sync operations. */ private val syncInterval: Duration = 30.seconds, /** - * Archived attachments can be used as a cache which can be restored if an attachment id + * Archived attachments can be used as a cache which can be restored if an attachment ID * reappears after being removed. This parameter defines how many archived records are retained. * Records are deleted once the number of items exceeds this value. */ private val archivedCacheLimit: Long = 100, /** - * Throttles remote sync operations triggering + * Throttles remote sync operations triggering. */ private val syncThrottleDuration: Duration = 1.seconds, /** - * Creates a list of subdirectories in the {attachmentDirectoryName} directory + * Creates a list of subdirectories in the [attachmentsDirectory] directory. */ private val subdirectories: List? = null, /** - * Should attachments be downloaded + * Should attachments be downloaded. */ private val downloadAttachments: Boolean = true, /** - * Logging interface used for all log operations + * Logging interface used for all log operations. */ public val logger: Logger = Logger, /** @@ -142,15 +145,22 @@ public open class AttachmentQueue private val coroutineScope: CoroutineScope? = null, ) { public companion object { + /** + * Default table name for attachments. + */ public const val DEFAULT_TABLE_NAME: String = "attachments" + + /** + * Default directory name for attachments. + */ public const val DEFAULT_ATTACHMENTS_DIRECTORY_NAME: String = "attachments" } /** * Service which provides access to attachment records. * Use this to: - * - Query all current attachment records - * - Create new attachment records for upload/download + * - Query all current attachment records. + * - Create new attachment records for upload/download. */ public val attachmentsService: AttachmentService = AttachmentServiceImpl( @@ -167,7 +177,7 @@ public open class AttachmentQueue /** * Syncing service for this attachment queue. - * This processes attachment records and performs relevant upload, download and delete + * This processes attachment records and performs relevant upload, download, and delete * operations. */ private val syncingService = @@ -185,10 +195,10 @@ public open class AttachmentQueue public var closed: Boolean = false /** - * Initialize the attachment queue by - * 1. Creating attachments directory - * 2. Adding watches for uploads, downloads, and deletes - * 3. Adding trigger to run uploads, downloads, and deletes when device is online after being offline + * Initialize the attachment queue by: + * 1. Creating the attachments directory. + * 2. Adding watches for uploads, downloads, and deletes. + * 3. Adding a trigger to run uploads, downloads, and deletes when the device is online after being offline. */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun startSync(): Unit = @@ -200,16 +210,20 @@ public open class AttachmentQueue stopSyncingInternal() - // Ensure the directory where attachments are downloaded, exists + // Ensure the directory where attachments are downloaded exists. localStorage.makeDir(attachmentsDirectory) subdirectories?.forEach { subdirectory -> localStorage.makeDir(Path(attachmentsDirectory, subdirectory).toString()) } + attachmentsService.withContext { context -> + verifyAttachments(context) + } + syncingService.startSync(syncInterval) - // Listen for connectivity changes + // Listen for connectivity changes. syncStatusJob = syncScope.launch { launch { @@ -223,8 +237,8 @@ public open class AttachmentQueue } launch { - // Watch local attachment relationships and sync the attachment records - watchedAttachments.collect { items -> + // Watch local attachment relationships and sync the attachment records. + watchAttachments().collect { items -> processWatchedAttachments(items) } } @@ -267,7 +281,7 @@ public open class AttachmentQueue syncStatusJob?.cancelAndJoin() syncingService.close() if (coroutineScope == null) { - // Only cancel the internal scope if we created it + // Only cancel the internal scope if we created it. syncScope.coroutineContext[Job]?.cancelAndJoin() } closed = true @@ -277,8 +291,8 @@ public open class AttachmentQueue /** * Resolves the filename for new attachment items. * A new attachment from [watchAttachments] might not include a filename. - * Concatenates the attachment ID an extension by default. - * This method can be overriden for custom behaviour. + * Concatenates the attachment ID and extension by default. + * This method can be overridden for custom behavior. */ @Throws(PowerSyncException::class, CancellationException::class) public open suspend fun resolveNewAttachmentFilename( @@ -288,7 +302,7 @@ public open class AttachmentQueue /** * Processes attachment items returned from [watchAttachments]. - * The default implementation assets the items returned from [watchAttachments] as the definitive + * The default implementation asserts the items returned from [watchAttachments] as the definitive * state for local attachments. * * Records currently in the attachment queue which are not present in the items are deleted from @@ -298,15 +312,15 @@ public open class AttachmentQueue * download. This requires that locally created attachments should be created with [saveFile] * before assigning the attachment ID to the relevant watched tables. * - * This method can be overriden for custom behaviour. + * This method can be overridden for custom behavior. */ @Throws(PowerSyncException::class, CancellationException::class) public open suspend fun processWatchedAttachments(items: List): Unit = runWrappedSuspending { /** - * Use a lock here to prevent conflicting state updates + * Use a lock here to prevent conflicting state updates. */ - attachmentsService.withLock { attachmentsContext -> + attachmentsService.withContext { attachmentsContext -> /** * Need to get all the attachments which are tracked in the DB. * We might need to restore an archived attachment. @@ -321,8 +335,8 @@ public open class AttachmentQueue if (!downloadAttachments) { continue } - // This item should be added to the queue - // This item is assumed to be coming from an upstream sync + // This item should be added to the queue. + // This item is assumed to be coming from an upstream sync. // Locally created new items should be persisted using [saveFile] before // this point. val filename = @@ -342,9 +356,9 @@ public open class AttachmentQueue } else if (existingQueueItem.state == AttachmentState.ARCHIVED) { // The attachment is present again. Need to queue it for sync. - // We might be able to optimize this in future + // We might be able to optimize this in future. if (existingQueueItem.hasSynced == 1) { - // No remote action required, we can restore the record (avoids deletion) + // No remote action required, we can restore the record (avoids deletion). attachmentUpdates.add( existingQueueItem.copy(state = AttachmentState.SYNCED), ) @@ -352,7 +366,7 @@ public open class AttachmentQueue /** * The localURI should be set if the record was meant to be downloaded * and has been synced. If it's missing and hasSynced is false then - * it must be an upload operation + * it must be an upload operation. */ attachmentUpdates.add( existingQueueItem.copy( @@ -390,10 +404,10 @@ public open class AttachmentQueue * The filename is resolved using [resolveNewAttachmentFilename]. * * A [updateHook] is provided which should be used when assigning relationships to the newly - * created attachment. This hook is executed in the same writeTransaction which creates the + * created attachment. This hook is executed in the same write transaction which creates the * attachment record. * - * This method can be overriden for custom behaviour. + * This method can be overridden for custom behavior. */ @Throws(PowerSyncException::class, CancellationException::class) public open suspend fun saveFile( @@ -409,14 +423,14 @@ public open class AttachmentQueue resolveNewAttachmentFilename(attachmentId = id, fileExtension = fileExtension) val localUri = getLocalUri(filename) - // write the file to the filesystem + // Write the file to the filesystem. val fileSize = localStorage.saveFile(localUri, data) /** * Starts a write transaction. The attachment record and relevant local relationship * assignment should happen in the same transaction. */ - attachmentsService.withLock { attachmentContext -> + attachmentsService.withContext { attachmentContext -> db.writeTransaction { tx -> val attachment = Attachment( @@ -430,7 +444,7 @@ public open class AttachmentQueue ) /** - * Allow consumers to set relationships to this attachment id + * Allow consumers to set relationships to this attachment ID. */ updateHook.invoke(tx, attachment) @@ -447,7 +461,7 @@ public open class AttachmentQueue * for delete. * The default implementation assumes the attachment record already exists locally. An exception * is thrown if the record does not exist locally. - * This method can be overriden for custom behaviour. + * This method can be overridden for custom behavior. */ @Throws(PowerSyncException::class, CancellationException::class) public open suspend fun deleteFile( @@ -455,7 +469,7 @@ public open class AttachmentQueue updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, ): Attachment = runWrappedSuspending { - attachmentsService.withLock { attachmentContext -> + attachmentsService.withContext { attachmentContext -> val attachment = attachmentContext.getAttachment(attachmentId) ?: throw error("Attachment record with id $attachmentId was not found.") @@ -471,17 +485,17 @@ public open class AttachmentQueue } /** - * Return users storage directory with the attachmentPath use to load the file. - * Example: filePath: "attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg" + * Returns the user's storage directory with the attachment path used to load the file. + * Example: filePath: "attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg". */ public open fun getLocalUri(filename: String): String = Path(attachmentsDirectory, filename).toString() /** - * Removes all archived items + * Removes all archived items. */ public suspend fun expireCache() { var done: Boolean - attachmentsService.withLock { context -> + attachmentsService.withContext { context -> do { done = syncingService.deleteArchivedAttachments(context) } while (!done) @@ -489,13 +503,40 @@ public open class AttachmentQueue } /** - * Clears the attachment queue and deletes all attachment files + * Clears the attachment queue and deletes all attachment files. */ public suspend fun clearQueue() { - attachmentsService.withLock { + attachmentsService.withContext { it.clearQueue() } - // Remove the attachments directory + // Remove the attachments directory. localStorage.rmDir(attachmentsDirectory) } + + /** + * Cleans up stale attachments. + */ + private suspend fun verifyAttachments(context: AttachmentContext) { + val attachments = context.getActiveAttachments() + val updates = mutableListOf() + + for (attachment in attachments) { + if (attachment.localUri == null) { + continue + } + val exists = localStorage.fileExists(attachment.localUri) + if ( + attachment.state == AttachmentState.SYNCED || attachment.state == AttachmentState.QUEUED_UPLOAD && !exists + ) { + updates.add( + attachment.copy( + state = AttachmentState.ARCHIVED, + localUri = null, + ), + ) + } + } + + context.saveAttachments(updates) + } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt index 13fe91c3..d1b013b0 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentService.kt @@ -84,5 +84,5 @@ public interface AttachmentService { * Executes a callback with an exclusive lock on all attachment operations. * This helps prevent race conditions between different updates. */ - public suspend fun withLock(action: suspend (context: AttachmentContext) -> R): R + public suspend fun withContext(action: suspend (context: AttachmentContext) -> R): R } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt index deae177e..5b54831f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentTable.kt @@ -5,7 +5,10 @@ import com.powersync.db.schema.ColumnType import com.powersync.db.schema.Table /** - * Creates a PowerSync table for storing local attachment state + * Creates a PowerSync table for storing local attachment state. + * + * @param name The name of the table. + * @return A [Table] object configured for storing attachment data. */ public fun createAttachmentsTable(name: String): Table = Table( diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorage.kt b/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorage.kt index 1ff6fef4..69ace829 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorage.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/LocalStorage.kt @@ -5,12 +5,17 @@ import kotlinx.coroutines.flow.Flow import kotlin.coroutines.cancellation.CancellationException /** - * Provides access to local storage on a device + * Provides access to local storage on a device. */ public interface LocalStorage { /** * Saves a source of data bytes to a path. - * @returns the bytesize of the file + * + * @param filePath The path where the file will be saved. + * @param data A [Flow] of [ByteArray] representing the file data. + * @return The byte size of the saved file. + * @throws PowerSyncException If an error occurs during the save operation. + * @throws CancellationException If the operation is cancelled. */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun saveFile( @@ -18,24 +23,70 @@ public interface LocalStorage { data: Flow, ): Long + /** + * Reads a file from the specified path. + * + * @param filePath The path of the file to read. + * @param mediaType Optional media type of the file. + * @return A [Flow] of [ByteArray] representing the file data. + * @throws PowerSyncException If an error occurs during the read operation. + * @throws CancellationException If the operation is cancelled. + */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun readFile( filePath: String, mediaType: String? = null, ): Flow + /** + * Deletes a file at the specified path. + * + * @param filePath The path of the file to delete. + * @throws PowerSyncException If an error occurs during the delete operation. + * @throws CancellationException If the operation is cancelled. + */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun deleteFile(filePath: String): Unit + /** + * Checks if a file exists at the specified path. + * + * @param filePath The path of the file to check. + * @return `true` if the file exists, `false` otherwise. + * @throws PowerSyncException If an error occurs during the check. + * @throws CancellationException If the operation is cancelled. + */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun fileExists(filePath: String): Boolean + /** + * Creates a directory at the specified path. + * + * @param path The path of the directory to create. + * @throws PowerSyncException If an error occurs during the directory creation. + * @throws CancellationException If the operation is cancelled. + */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun makeDir(path: String): Unit + /** + * Removes a directory at the specified path. + * + * @param path The path of the directory to remove. + * @throws PowerSyncException If an error occurs during the directory removal. + * @throws CancellationException If the operation is cancelled. + */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun rmDir(path: String): Unit + /** + * Copies a file from the source path to the target path. + * + * @param sourcePath The path of the source file. + * @param targetPath The path where the file will be copied to. + * @throws PowerSyncException If an error occurs during the copy operation. + * @throws CancellationException If the operation is cancelled. + */ @Throws(PowerSyncException::class, CancellationException::class) public suspend fun copyFile( sourcePath: String, diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/README.md b/core/src/commonMain/kotlin/com/powersync/attachments/README.md index 197056f8..66b6de8d 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/README.md +++ b/core/src/commonMain/kotlin/com/powersync/attachments/README.md @@ -58,14 +58,16 @@ val queue = AttachmentQueue( db = db, attachmentsDirectory = attachmentsDirectory, remoteStorage = SupabaseRemoteStorage(supabase), - watchedAttachments = db.watch( - sql = """ - SELECT photo_id - FROM checklists - WHERE photo_id IS NOT NULL - """, - ) { - WatchedAttachmentItem(id = it.getString("photo_id"), fileExtension = "jpg") + watchAttachments = { + db.watch( + sql = """ + SELECT photo_id + FROM checklists + WHERE photo_id IS NOT NULL + """, + ) { + WatchedAttachmentItem(id = it.getString("photo_id"), fileExtension = "jpg") + } } ) ``` @@ -173,7 +175,7 @@ The `AttachmentQueue` implements a sync process with these components: 2. **Periodic Sync**: By default, the queue triggers a sync every 30 seconds to retry failed uploads/downloads, in particular after the app was offline. This interval can be configured by setting `syncInterval` in the `AttachmentQueue` constructor options, or disabled by setting the interval to `0`. -3. **Watching State**: The `watchedAttachments` flow in the `AttachmentQueue` constructor is used to maintain consistency between local and remote states: +3. **Watching State**: The `watchAttachments` flow generator in the `AttachmentQueue` constructor is used to maintain consistency between local and remote states: - New items trigger downloads - see the Download Process below. - Missing items trigger archiving - see Cache Management below. @@ -190,7 +192,7 @@ The `saveFile` method handles attachment creation and upload: ### Download Process -Attachments are scheduled for download when the `watchedAttachments` flow emits a new item that is not present locally: +Attachments are scheduled for download when the flow from `watchAttachments` emits a new item that is not present locally: 1. An `AttachmentRecord` is created with `QUEUED_DOWNLOAD` state 2. The `RemoteStorage` `downloadFile` function is called @@ -211,7 +213,7 @@ The `deleteFile` method deletes attachments from both local and remote storage: The `AttachmentQueue` implements a caching system for archived attachments: -1. Local attachments are marked as `ARCHIVED` if the `watchedAttachments` flow no longer references them +1. Local attachments are marked as `ARCHIVED` if the flow from `watchAttachments` no longer references them 2. Archived attachments are kept in the cache for potential future restoration 3. The cache size is controlled by the `archivedCacheLimit` parameter in the `AttachmentQueue` constructor 4. By default, the queue keeps the last 100 archived attachment records diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorage.kt b/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorage.kt index 8035d94e..0a700224 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorage.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/RemoteStorage.kt @@ -7,7 +7,10 @@ import kotlinx.coroutines.flow.Flow */ public interface RemoteStorage { /** - * Upload a file to remote storage + * Uploads a file to remote storage. + * + * @param fileData The file data as a flow of byte arrays. + * @param attachment The attachment record associated with the file. */ public suspend fun uploadFile( fileData: Flow, @@ -15,12 +18,17 @@ public interface RemoteStorage { ): Unit /** - * Download a file from remote storage + * Downloads a file from remote storage. + * + * @param attachment The attachment record associated with the file. + * @return A flow of byte arrays representing the file data. */ public suspend fun downloadFile(attachment: Attachment): Flow /** - * Delete a file from remote storage + * Deletes a file from remote storage. + * + * @param attachment The attachment record associated with the file. */ public suspend fun deleteFile(attachment: Attachment) } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/SyncErrorHandler.kt b/core/src/commonMain/kotlin/com/powersync/attachments/SyncErrorHandler.kt index 3d8b262c..f32b9193 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/SyncErrorHandler.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/SyncErrorHandler.kt @@ -1,13 +1,17 @@ package com.powersync.attachments /** - * Handles attachment operation errors. - * The handlers here specify if the corresponding operations should be retried. - * Attachment records are archived if an operation failed and should not be retried. + * Interface for handling errors during attachment operations. + * Implementations determine whether failed operations should be retried. + * Attachment records are archived if an operation fails and should not be retried. */ public interface SyncErrorHandler { /** - * @returns if the provided attachment download operation should be retried + * Determines whether the provided attachment download operation should be retried. + * + * @param attachment The attachment involved in the failed download operation. + * @param exception The exception that caused the download failure. + * @return `true` if the download operation should be retried, `false` otherwise. */ public suspend fun onDownloadError( attachment: Attachment, @@ -15,7 +19,11 @@ public interface SyncErrorHandler { ): Boolean /** - * @returns if the provided attachment upload operation should be retried + * Determines whether the provided attachment upload operation should be retried. + * + * @param attachment The attachment involved in the failed upload operation. + * @param exception The exception that caused the upload failure. + * @return `true` if the upload operation should be retried, `false` otherwise. */ public suspend fun onUploadError( attachment: Attachment, @@ -23,7 +31,11 @@ public interface SyncErrorHandler { ): Boolean /** - * @returns if the provided attachment delete operation should be retried + * Determines whether the provided attachment delete operation should be retried. + * + * @param attachment The attachment involved in the failed delete operation. + * @param exception The exception that caused the delete failure. + * @return `true` if the delete operation should be retried, `false` otherwise. */ public suspend fun onDeleteError( attachment: Attachment, diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt index b48453f6..a013111f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt @@ -5,6 +5,7 @@ import com.powersync.PowerSyncDatabase import com.powersync.attachments.Attachment import com.powersync.attachments.AttachmentContext import com.powersync.attachments.AttachmentState +import com.powersync.db.getString import com.powersync.db.internal.ConnectionContext import kotlinx.datetime.Clock import kotlinx.serialization.encodeToString @@ -70,7 +71,7 @@ public open class AttachmentContextImpl( public override suspend fun getAttachmentIds(): List = db.getAll( "SELECT id FROM $table WHERE id IS NOT NULL", - ) { it.getString(0)!! } + ) { it.getString("name") } public override suspend fun getAttachments(): List = db.getAll( @@ -92,16 +93,16 @@ public open class AttachmentContextImpl( public override suspend fun getActiveAttachments(): List = db.getAll( """ - SELECT - * - FROM - $table - WHERE - state = ? - OR state = ? - OR state = ? - ORDER BY - timestamp ASC + SELECT + * + FROM + $table + WHERE + state = ? + OR state = ? + OR state = ? + ORDER BY + timestamp ASC """, listOf( AttachmentState.QUEUED_UPLOAD.ordinal, @@ -130,15 +131,15 @@ public open class AttachmentContextImpl( val attachments = db.getAll( """ - SELECT - * - FROM - $table - WHERE - state = ? - ORDER BY - timestamp DESC - LIMIT ? OFFSET ? + SELECT + * + FROM + $table + WHERE + state = ? + ORDER BY + timestamp DESC + LIMIT ? OFFSET ? """, listOf( AttachmentState.ARCHIVED.ordinal, @@ -171,9 +172,9 @@ public open class AttachmentContextImpl( context.execute( """ INSERT OR REPLACE INTO - $table (id, timestamp, filename, local_uri, media_type, size, state, has_synced) + $table (id, timestamp, filename, local_uri, media_type, size, state, has_synced, meta_data) VALUES - (?, ?, ?, ?, ?, ?, ?, ?) + (?, ?, ?, ?, ?, ?, ?, ?, ?) """, listOf( updatedRecord.id, @@ -184,6 +185,7 @@ public open class AttachmentContextImpl( updatedRecord.size, updatedRecord.state.ordinal, updatedRecord.hasSynced, + updatedRecord.metaData, ), ) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt index 03a7e584..c94c09eb 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt @@ -5,6 +5,7 @@ import com.powersync.PowerSyncDatabase import com.powersync.attachments.AttachmentContext import com.powersync.attachments.AttachmentService import com.powersync.attachments.AttachmentState +import com.powersync.db.getString import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.sync.Mutex @@ -35,7 +36,7 @@ public open class AttachmentServiceImpl( maxArchivedCount = maxArchivedCount, ) - public override suspend fun withLock(action: suspend (AttachmentContext) -> R): R = mutex.withLock { action(context) } + public override suspend fun withContext(action: suspend (AttachmentContext) -> R): R = mutex.withLock { action(context) } /** * Watcher for changes to attachments table. @@ -62,7 +63,7 @@ public open class AttachmentServiceImpl( AttachmentState.QUEUED_DOWNLOAD.ordinal, AttachmentState.QUEUED_DELETE.ordinal, ), - ) { it.getString(0)!! } + ) { it.getString("id") } // We only use changes here to trigger a sync consolidation .map { Unit } } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index de32f621..cd9af5a7 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -89,7 +89,7 @@ public class SyncingService .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) .throttle(syncThrottle) .collect { - attachmentsService.withLock { context -> + attachmentsService.withContext { context -> /** * Gets and performs the operations for active attachments which are * pending download, upload, or delete. diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index af82b3b3..70120158 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -55,7 +55,7 @@ fun App( db = db, remoteStorage = SupabaseRemoteStorage(supabase), attachmentsDirectory = attachmentDirectory, - watchedAttachments = + watchAttachments = { db.watch( "SELECT photo_id from todos WHERE photo_id IS NOT NULL", ) { @@ -63,7 +63,8 @@ fun App( id = it.getString("photo_id"), fileExtension = "jpg", ) - }, + } + } ) } else { null From 7c5152dca985105202a924b85ed0c62718ed3e17 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 12:44:39 +0200 Subject: [PATCH 42/51] add changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 682443d1..040623d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.0.0-BETA31 + +* Added helpers for Attachment syncing. + ## 1.0.0-BETA30 * Fix a deadlock when calling `connect()` immediately after opening a database. From b4d43262db6cb8e643ec3c0b5d6eeacb969a4f92 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 12:51:04 +0200 Subject: [PATCH 43/51] cleanup --- .../kotlin/com/powersync/AttachmentsTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 2b958be3..5305a02f 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -198,7 +198,7 @@ class AttachmentsTest { db = database, remoteStorage = remote, attachmentsDirectory = getAttachmentsDir(), - watchAttachments = { watchAttachments(database) }, + watchAttachments = { watchAttachments(database) }, /** * Sets the cache limit to zero for this test. Archived records will * immediately be deleted. @@ -407,7 +407,7 @@ class AttachmentsTest { db = database, remoteStorage = remote, attachmentsDirectory = getAttachmentsDir(), - watchAttachments = { watchAttachments(database) }, + watchAttachments = { watchAttachments(database) }, /** * Keep some items in the cache */ From f4e9b78e9f532082d3b4cff30178dd633d30e4bc Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 14:23:47 +0200 Subject: [PATCH 44/51] test --- .../kotlin/com/powersync/AttachmentsTest.kt | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 5305a02f..a3847b60 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -224,32 +224,37 @@ class AttachmentsTest { * Creates an attachment given a flow of bytes (the file data) then assigns this to * a newly created user. */ - queue.saveFile( - data = flowOf(ByteArray(1)), - mediaType = "image/jpg", - fileExtension = "jpg", - ) { tx, attachment -> - // Set the photo_id of a new user to the attachment id - tx.execute( - // language=SQL - """ - INSERT INTO - users (id, name, email, photo_id) - VALUES - (uuid(), "steven", "steven@steven.com", ?) + val record = + queue.saveFile( + data = flowOf(ByteArray(1)), + mediaType = "image/jpg", + fileExtension = "jpg", + ) { tx, attachment -> + // Set the photo_id of a new user to the attachment id + tx.execute( + // language=SQL + """ + INSERT INTO + users (id, name, email, photo_id) + VALUES + (uuid(), "steven", "steven@steven.com", ?) """, - listOf(attachment.id), - ) - } + listOf(attachment.id), + ) + } + + println("Record: $record") var attachmentRecord = attachmentQuery.awaitItem().first() attachmentRecord shouldNotBe null if (attachmentRecord.state == AttachmentState.QUEUED_UPLOAD) { // Wait for it to be synced + println(attachmentRecord) attachmentRecord = attachmentQuery.awaitItem().first() } + println(attachmentRecord) attachmentRecord.state shouldBe AttachmentState.SYNCED // A download should have been attempted for this file From 5fd3f8450fdef76dee3bce550f5a64ce8d393710 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 15:41:11 +0200 Subject: [PATCH 45/51] test --- .../kotlin/com/powersync/AttachmentsTest.kt | 4 +- .../powersync/attachments/AttachmentQueue.kt | 5 ++- .../com/powersync/attachments/FIFOLock.kt | 43 +++++++++++++++++++ .../implementation/AttachmentServiceImpl.kt | 4 +- 4 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 core/src/commonMain/kotlin/com/powersync/attachments/FIFOLock.kt diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index a3847b60..b5c64df5 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -96,8 +96,8 @@ class AttachmentsTest { ) doOnCleanup { - queue.stopSyncing() attachmentQuery.cancel() + queue.stopSyncing() queue.clearQueue() queue.close() } @@ -207,10 +207,10 @@ class AttachmentsTest { ) doOnCleanup { + attachmentQuery.cancel() queue.stopSyncing() queue.clearQueue() queue.close() - attachmentQuery.cancel() } queue.startSync() diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 3799357c..01f5d858 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -6,6 +6,7 @@ import com.powersync.PowerSyncException import com.powersync.attachments.implementation.AttachmentServiceImpl import com.powersync.attachments.storage.IOLocalStorageAdapter import com.powersync.attachments.sync.SyncingService +import com.powersync.db.getString import com.powersync.db.internal.ConnectionContext import com.powersync.db.runWrappedSuspending import kotlinx.coroutines.CoroutineScope @@ -262,6 +263,7 @@ public open class AttachmentQueue } syncStatusJob?.cancelAndJoin() + syncStatusJob = null syncingService.stopSync() } @@ -390,6 +392,7 @@ public open class AttachmentQueue it.state != AttachmentState.QUEUED_DELETE && null == items.find { update -> update.id == it.id } }.forEach { + println("Archiving attachment ${it.id}") attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED)) } @@ -418,7 +421,7 @@ public open class AttachmentQueue updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, ): Attachment = runWrappedSuspending { - val id = db.get("SELECT uuid()") { it.getString(0)!! } + val id = db.get("SELECT uuid() as id") { it.getString("id") } val filename = resolveNewAttachmentFilename(attachmentId = id, fileExtension = fileExtension) val localUri = getLocalUri(filename) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/FIFOLock.kt b/core/src/commonMain/kotlin/com/powersync/attachments/FIFOLock.kt new file mode 100644 index 00000000..6a452512 --- /dev/null +++ b/core/src/commonMain/kotlin/com/powersync/attachments/FIFOLock.kt @@ -0,0 +1,43 @@ +package com.powersync.attachments + +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +internal class FIFOLock { + private val mutex = Mutex() + private val queue = mutableListOf>() + + suspend fun lock() { + val deferred = CompletableDeferred() + + mutex.withLock { + queue.add(deferred) + // If this is the only request, grant the lock immediately + if (queue.size == 1) { + deferred.complete(Unit) + } + } + + // Suspend until the lock is granted + deferred.await() + } + + suspend fun withLock(action: suspend () -> R): R { + lock() + return try { + action() + } finally { + unlock() + } + } + + suspend fun unlock() { + mutex.withLock { + // Remove the current lock holder + queue.removeAt(0) + // Grant the lock to the next request in the queue, if any + queue.firstOrNull()?.complete(Unit) + } + } +} diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt index c94c09eb..d1243b77 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt @@ -5,10 +5,10 @@ import com.powersync.PowerSyncDatabase import com.powersync.attachments.AttachmentContext import com.powersync.attachments.AttachmentService import com.powersync.attachments.AttachmentState +import com.powersync.attachments.FIFOLock import com.powersync.db.getString import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map -import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock /** @@ -26,7 +26,7 @@ public open class AttachmentServiceImpl( private val table: String get() = tableName - private val mutex = Mutex() + private val mutex = FIFOLock() private val context: AttachmentContext = AttachmentContextImpl( From 39ea64960d36e9d93a6b9b3fbeda899f3cbae96b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 15:59:50 +0200 Subject: [PATCH 46/51] test --- .../kotlin/com/powersync/attachments/AttachmentQueue.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 01f5d858..2dcf957c 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -390,6 +390,7 @@ public open class AttachmentQueue currentAttachments .filter { it.state != AttachmentState.QUEUED_DELETE && + it.state != AttachmentState.QUEUED_UPLOAD && null == items.find { update -> update.id == it.id } }.forEach { println("Archiving attachment ${it.id}") From d1760c520f67429da94981c43ff29d5656b511f6 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 16:13:09 +0200 Subject: [PATCH 47/51] test --- .../kotlin/com/powersync/AttachmentsTest.kt | 4 ---- .../kotlin/com/powersync/attachments/AttachmentQueue.kt | 1 - .../attachments/implementation/AttachmentServiceImpl.kt | 4 ++-- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index b5c64df5..151e8560 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -243,18 +243,14 @@ class AttachmentsTest { ) } - println("Record: $record") - var attachmentRecord = attachmentQuery.awaitItem().first() attachmentRecord shouldNotBe null if (attachmentRecord.state == AttachmentState.QUEUED_UPLOAD) { // Wait for it to be synced - println(attachmentRecord) attachmentRecord = attachmentQuery.awaitItem().first() } - println(attachmentRecord) attachmentRecord.state shouldBe AttachmentState.SYNCED // A download should have been attempted for this file diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 2dcf957c..8c897086 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -393,7 +393,6 @@ public open class AttachmentQueue it.state != AttachmentState.QUEUED_UPLOAD && null == items.find { update -> update.id == it.id } }.forEach { - println("Archiving attachment ${it.id}") attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED)) } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt index d1243b77..c94c09eb 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt @@ -5,10 +5,10 @@ import com.powersync.PowerSyncDatabase import com.powersync.attachments.AttachmentContext import com.powersync.attachments.AttachmentService import com.powersync.attachments.AttachmentState -import com.powersync.attachments.FIFOLock import com.powersync.db.getString import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map +import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock /** @@ -26,7 +26,7 @@ public open class AttachmentServiceImpl( private val table: String get() = tableName - private val mutex = FIFOLock() + private val mutex = Mutex() private val context: AttachmentContext = AttachmentContextImpl( From 5b2f59008357429c03366e1c1bd12243d9c19242 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 17 Apr 2025 17:01:47 +0200 Subject: [PATCH 48/51] cleanup changelog --- CHANGELOG.md | 76 ++++++++++++------- .../com/powersync/attachments/FIFOLock.kt | 43 ----------- 2 files changed, 50 insertions(+), 69 deletions(-) delete mode 100644 core/src/commonMain/kotlin/com/powersync/attachments/FIFOLock.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 040623d7..94996f76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,23 +14,31 @@ * Fix potential race condition between jobs in `connect()` and `disconnect()`. * [JVM Windows] Fixed PowerSync Extension temporary file deletion error on process shutdown. * [iOS] Fixed issue where automatic driver migrations would fail with the error: + ``` Sqlite operation failure database is locked attempted to run migration and failed. closing connection ``` + * Fix race condition causing data received during uploads not to be applied. -* Added Attachment helpers ## 1.0.0-BETA28 * Update PowerSync SQLite core extension to 0.3.12. -* Added queing protection and warnings when connecting multiple PowerSync clients to the same database file. -* Improved concurrent SQLite connection support accross various platforms. All platforms now use a single write connection and multiple read connections for concurrent read queries. -* Added the ability to open a SQLite database given a custom `dbDirectory` path. This is currently not supported on iOS due to internal driver restrictions. +* Added queing protection and warnings when connecting multiple PowerSync clients to the same + database file. +* Improved concurrent SQLite connection support accross various platforms. All platforms now use a + single write connection and multiple read connections for concurrent read queries. +* Added the ability to open a SQLite database given a custom `dbDirectory` path. This is currently + not supported on iOS due to internal driver restrictions. * Internaly improved the linking of SQLite for iOS. * Enabled Full Text Search on iOS platforms. * Added the ability to update the schema for existing PowerSync clients. -* Fixed bug where local only, insert only and view name overrides were not applied for schema tables. -* The Android SQLite driver now uses the [Xerial JDBC library](https://github.com/xerial/sqlite-jdbc). This removes the requirement for users to add the jitpack Maven repository to their projects. +* Fixed bug where local only, insert only and view name overrides were not applied for schema + tables. +* The Android SQLite driver now uses + the [Xerial JDBC library](https://github.com/xerial/sqlite-jdbc). This removes the requirement for + users to add the jitpack Maven repository to their projects. + ```diff // settings.gradle.kts example repositories { @@ -58,8 +66,10 @@ Sqlite operation failure database is locked attempted to run migration and faile ## 1.0.0-BETA24 -* Improve internal handling of watch queries to avoid issues where updates are not being received due to transaction commits occurring after the query is run. -* Fix issue in JVM build where `columnNames` was throwing an error due to the index of the JDBC driver starting at 1 instead of 0 as in the other drivers/ +* Improve internal handling of watch queries to avoid issues where updates are not being received + due to transaction commits occurring after the query is run. +* Fix issue in JVM build where `columnNames` was throwing an error due to the index of the JDBC + driver starting at 1 instead of 0 as in the other drivers/ * Throw and not just catch `CancellationExceptions` in `runWrappedSuspending` ## 1.0.0-BETA23 @@ -77,14 +87,18 @@ Sqlite operation failure database is locked attempted to run migration and faile ## 1.0.0-BETA20 -* Add cursor optional functions: `getStringOptional`, `getLongOptional`, `getDoubleOptional`, `getBooleanOptional` and `getBytesOptional` when using the column name which allow for optional return types +* Add cursor optional functions: `getStringOptional`, `getLongOptional`, `getDoubleOptional`, + `getBooleanOptional` and `getBytesOptional` when using the column name which allow for optional + return types * Throw errors for invalid column on all cursor functions -* `getString`, `getLong`, `getBytes`, `getDouble` and `getBoolean` used with the column name will now throw an error for non-null values and expect a non optional return type +* `getString`, `getLong`, `getBytes`, `getDouble` and `getBoolean` used with the column name will + now throw an error for non-null values and expect a non optional return type ## 1.0.0-BETA19 * Allow cursor to get values by column name e.g. `getStringOptional("id")` -* BREAKING CHANGE: If you were using `SqlCursor` from SqlDelight previously for your own custom mapper then you must now change to `SqlCursor` exported by the PowerSync module. +* BREAKING CHANGE: If you were using `SqlCursor` from SqlDelight previously for your own custom + mapper then you must now change to `SqlCursor` exported by the PowerSync module. Previously you would import it like this: @@ -100,7 +114,8 @@ Sqlite operation failure database is locked attempted to run migration and faile ## 1.0.0-BETA18 -* BREAKING CHANGE: Move from async sqldelight calls to synchronous calls. This will only affect `readTransaction` and `writeTransaction`where the callback function is no longer asynchronous. +* BREAKING CHANGE: Move from async sqldelight calls to synchronous calls. This will only affect + `readTransaction` and `writeTransaction`where the callback function is no longer asynchronous. ## 1.0.0-BETA17 @@ -109,7 +124,8 @@ Sqlite operation failure database is locked attempted to run migration and faile ## 1.0.0-BETA16 * Add `close` method to database methods -* Throw when error is a `CancellationError` and remove invalidation for all errors in `streamingSync` catch. +* Throw when error is a `CancellationError` and remove invalidation for all errors in + `streamingSync` catch. ## 1.0.0-BETA15 @@ -123,7 +139,8 @@ Sqlite operation failure database is locked attempted to run migration and faile ## 1.0.0-BETA13 -* Move iOS database driver to use IO dispatcher which should avoid race conditions and improve performance. +* Move iOS database driver to use IO dispatcher which should avoid race conditions and improve + performance. ## 1.0.0-BETA12 @@ -140,7 +157,8 @@ Sqlite operation failure database is locked attempted to run migration and faile ## 1.0.0-BETA9 * Re-enable SKIE `SuspendInterop` -* Move transaction functions out of `PowerSyncTransactionFactory` to avoid threading issues in Swift SDK +* Move transaction functions out of `PowerSyncTransactionFactory` to avoid threading issues in Swift + SDK ## 1.0.0-BETA8 @@ -169,23 +187,27 @@ Sqlite operation failure database is locked attempted to run migration and faile * Add `waitForFirstSync` function - which resolves after the initial sync is completed * Upgrade to Kotlin 2.0.20 - should not cause any issues with users who are still on Kotlin 1.9 * Upgrade `powersync-sqlite-core` to 0.3.0 - improves incremental sync performance -* Add client sync parameters - which allows you specify sync parameters from the client https://docs.powersync.com/usage/sync-rules/advanced-topics/client-parameters-beta +* Add client sync parameters - which allows you specify sync parameters from the + client https://docs.powersync.com/usage/sync-rules/advanced-topics/client-parameters-beta + ```kotlin val params = JsonParam.Map( - mapOf( - "name" to JsonParam.String("John Doe"), - "age" to JsonParam.Number(30), - "isStudent" to JsonParam.Boolean(false) - ) + mapOf( + "name" to JsonParam.String("John Doe"), + "age" to JsonParam.Number(30), + "isStudent" to JsonParam.Boolean(false) + ) ) connect( -... - params = params + ... +params = params ) ``` + * Add schema validation when schema is generated -* Add warning message if there is a crudItem in the queue that has not yet been synced and after a delay rerun the upload +* Add warning message if there is a crudItem in the queue that has not yet been synced and after a + delay rerun the upload ## 1.0.0-BETA2 @@ -193,13 +215,15 @@ connect( ## 1.0.0-BETA1 -* Improve API by changing from Builder pattern to simply instantiating the database `PowerSyncDatabase` +* Improve API by changing from Builder pattern to simply instantiating the database + `PowerSyncDatabase` E.g. `val db = PowerSyncDatabase(factory, schema)` * Use callback context in transactions E.g. `db.writeTransaction{ ctx -> ctx.execute(...) }` * Removed unnecessary expiredAt field * Added table max column validation as there is a hard limit of 63 columns * Moved SQLDelight models to a separate module to reduce export size -* Replaced default Logger with [Kermit Logger](https://kermit.touchlab.co/) which allows users to more easily use and/or change Logger settings +* Replaced default Logger with [Kermit Logger](https://kermit.touchlab.co/) which allows users to + more easily use and/or change Logger settings * Add `retryDelay` and `crudThrottle` options when setting up database connection * Changed `_viewNameOverride` to `viewNameOverride` diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/FIFOLock.kt b/core/src/commonMain/kotlin/com/powersync/attachments/FIFOLock.kt deleted file mode 100644 index 6a452512..00000000 --- a/core/src/commonMain/kotlin/com/powersync/attachments/FIFOLock.kt +++ /dev/null @@ -1,43 +0,0 @@ -package com.powersync.attachments - -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock - -internal class FIFOLock { - private val mutex = Mutex() - private val queue = mutableListOf>() - - suspend fun lock() { - val deferred = CompletableDeferred() - - mutex.withLock { - queue.add(deferred) - // If this is the only request, grant the lock immediately - if (queue.size == 1) { - deferred.complete(Unit) - } - } - - // Suspend until the lock is granted - deferred.await() - } - - suspend fun withLock(action: suspend () -> R): R { - lock() - return try { - action() - } finally { - unlock() - } - } - - suspend fun unlock() { - mutex.withLock { - // Remove the current lock holder - queue.removeAt(0) - // Grant the lock to the next request in the queue, if any - queue.firstOrNull()?.complete(Unit) - } - } -} From 4208b9e2f31967bdc1933223ce5568eb589f8e94 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 22 Apr 2025 10:21:16 +0200 Subject: [PATCH 49/51] cleanup coroutine scopes --- .../com/powersync/attachments/AttachmentQueue.kt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 8c897086..c27b98e4 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -10,7 +10,6 @@ import com.powersync.db.getString import com.powersync.db.internal.ConnectionContext import com.powersync.db.runWrappedSuspending import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO import kotlinx.coroutines.Job @@ -61,7 +60,6 @@ public data class WatchedAttachmentItem( * determine which folder attachments are stored into. */ public open class AttachmentQueue - @OptIn(DelicateCoroutinesApi::class) constructor( /** * PowerSync database client. @@ -171,7 +169,14 @@ public open class AttachmentQueue maxArchivedCount = archivedCacheLimit, ) - private val syncScope = coroutineScope ?: CoroutineScope(SupervisorJob() + Dispatchers.IO) + // The syncScope is used to manage coroutines for syncing operations. + // If a coroutineScope is provided, it reuses its context and adds Dispatchers.IO for IO-bound tasks, + // ensuring proper cancellation propagation. If no coroutineScope is provided, a new CoroutineScope + // is created with a SupervisorJob to isolate failures and Dispatchers.IO for efficient IO operations. + private val syncScope = + coroutineScope?.let { + CoroutineScope(it.coroutineContext + Dispatchers.IO) + } ?: CoroutineScope(CoroutineScope(Dispatchers.IO).coroutineContext + SupervisorJob()) private var syncStatusJob: Job? = null private val mutex = Mutex() @@ -284,7 +289,7 @@ public open class AttachmentQueue syncingService.close() if (coroutineScope == null) { // Only cancel the internal scope if we created it. - syncScope.coroutineContext[Job]?.cancelAndJoin() + syncScope.coroutineContext[Job]?.takeIf { it.isActive }?.cancelAndJoin() } closed = true } From e06be2bd92b57a592d557fe21170835593c03c22 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 22 Apr 2025 10:35:42 +0200 Subject: [PATCH 50/51] Update comments on open classes --- .../powersync/attachments/AttachmentQueue.kt | 851 +++++++++--------- .../implementation/AttachmentContextImpl.kt | 7 + .../implementation/AttachmentServiceImpl.kt | 5 +- .../storage/IOLocalStorageAdapter.kt | 6 +- .../attachments/sync/SyncingService.kt | 450 ++++----- .../java/com/powersync/androidexample/App.kt | 4 +- .../local.properties.example | 3 +- 7 files changed, 668 insertions(+), 658 deletions(-) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index c27b98e4..2705159b 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -59,492 +59,491 @@ public data class WatchedAttachmentItem( * AbstractRemoteStorageAdapter, and an attachment directory name which will * determine which folder attachments are stored into. */ -public open class AttachmentQueue - constructor( - /** - * PowerSync database client. - */ - public val db: PowerSyncDatabase, - /** - * Adapter which interfaces with the remote storage backend. - */ - public val remoteStorage: RemoteStorage, - /** - * Directory name where attachment files will be written to disk. - * This will be created if it does not exist. - */ - private val attachmentsDirectory: String, - /** - * A flow generator for the current state of local attachments. - * Example: - * ```kotlin - * watchAttachments = { - * db.watch( - * sql = """ - * SELECT - * photo_id as id, - * 'jpg' as fileExtension - * FROM - * checklists - * WHERE - * photo_id IS NOT NULL - * """, - * ) { cursor -> - * WatchedAttachmentItem( - * id = cursor.getString("id"), - * fileExtension = "jpg", - * ) - * } - * } - * ``` - */ - private val watchAttachments: () -> Flow>, - /** - * Provides access to local filesystem storage methods. - */ - public val localStorage: LocalStorage = IOLocalStorageAdapter(), - /** - * SQLite table where attachment state will be recorded. - */ - private val attachmentsQueueTableName: String = DEFAULT_TABLE_NAME, - /** - * Attachment operation error handler. Specifies if failed attachment operations - * should be retried. - */ - private val errorHandler: SyncErrorHandler? = null, - /** - * Periodic interval to trigger attachment sync operations. - */ - private val syncInterval: Duration = 30.seconds, - /** - * Archived attachments can be used as a cache which can be restored if an attachment ID - * reappears after being removed. This parameter defines how many archived records are retained. - * Records are deleted once the number of items exceeds this value. - */ - private val archivedCacheLimit: Long = 100, - /** - * Throttles remote sync operations triggering. - */ - private val syncThrottleDuration: Duration = 1.seconds, - /** - * Creates a list of subdirectories in the [attachmentsDirectory] directory. - */ - private val subdirectories: List? = null, - /** - * Should attachments be downloaded. - */ - private val downloadAttachments: Boolean = true, +public open class AttachmentQueue( + /** + * PowerSync database client. + */ + public val db: PowerSyncDatabase, + /** + * Adapter which interfaces with the remote storage backend. + */ + public val remoteStorage: RemoteStorage, + /** + * Directory name where attachment files will be written to disk. + * This will be created if it does not exist. + */ + private val attachmentsDirectory: String, + /** + * A flow generator for the current state of local attachments. + * Example: + * ```kotlin + * watchAttachments = { + * db.watch( + * sql = """ + * SELECT + * photo_id as id, + * 'jpg' as fileExtension + * FROM + * checklists + * WHERE + * photo_id IS NOT NULL + * """, + * ) { cursor -> + * WatchedAttachmentItem( + * id = cursor.getString("id"), + * fileExtension = "jpg", + * ) + * } + * } + * ``` + */ + private val watchAttachments: () -> Flow>, + /** + * Provides access to local filesystem storage methods. + */ + public val localStorage: LocalStorage = IOLocalStorageAdapter(), + /** + * SQLite table where attachment state will be recorded. + */ + private val attachmentsQueueTableName: String = DEFAULT_TABLE_NAME, + /** + * Attachment operation error handler. Specifies if failed attachment operations + * should be retried. + */ + private val errorHandler: SyncErrorHandler? = null, + /** + * Periodic interval to trigger attachment sync operations. + */ + private val syncInterval: Duration = 30.seconds, + /** + * Archived attachments can be used as a cache which can be restored if an attachment ID + * reappears after being removed. This parameter defines how many archived records are retained. + * Records are deleted once the number of items exceeds this value. + */ + private val archivedCacheLimit: Long = 100, + /** + * Throttles remote sync operations triggering. + */ + private val syncThrottleDuration: Duration = 1.seconds, + /** + * Creates a list of subdirectories in the [attachmentsDirectory] directory. + */ + private val subdirectories: List? = null, + /** + * Should attachments be downloaded. + */ + private val downloadAttachments: Boolean = true, + /** + * Logging interface used for all log operations. + */ + public val logger: Logger = Logger, + /** + * Optional scope to launch syncing jobs in. + */ + private val coroutineScope: CoroutineScope? = null, +) { + public companion object { /** - * Logging interface used for all log operations. + * Default table name for attachments. */ - public val logger: Logger = Logger, + public const val DEFAULT_TABLE_NAME: String = "attachments" + /** - * Optional scope to launch syncing jobs in. + * Default directory name for attachments. */ - private val coroutineScope: CoroutineScope? = null, - ) { - public companion object { - /** - * Default table name for attachments. - */ - public const val DEFAULT_TABLE_NAME: String = "attachments" + public const val DEFAULT_ATTACHMENTS_DIRECTORY_NAME: String = "attachments" + } - /** - * Default directory name for attachments. - */ - public const val DEFAULT_ATTACHMENTS_DIRECTORY_NAME: String = "attachments" - } + /** + * Service which provides access to attachment records. + * Use this to: + * - Query all current attachment records. + * - Create new attachment records for upload/download. + */ + public val attachmentsService: AttachmentService = + AttachmentServiceImpl( + db, + attachmentsQueueTableName, + logger, + maxArchivedCount = archivedCacheLimit, + ) + + // The syncScope is used to manage coroutines for syncing operations. + // If a coroutineScope is provided, it reuses its context and adds Dispatchers.IO for IO-bound tasks, + // ensuring proper cancellation propagation. If no coroutineScope is provided, a new CoroutineScope + // is created with a SupervisorJob to isolate failures and Dispatchers.IO for efficient IO operations. + private val syncScope = + coroutineScope?.let { + CoroutineScope(it.coroutineContext + Dispatchers.IO) + } ?: CoroutineScope(CoroutineScope(Dispatchers.IO).coroutineContext + SupervisorJob()) + + private var syncStatusJob: Job? = null + private val mutex = Mutex() - /** - * Service which provides access to attachment records. - * Use this to: - * - Query all current attachment records. - * - Create new attachment records for upload/download. - */ - public val attachmentsService: AttachmentService = - AttachmentServiceImpl( - db, - attachmentsQueueTableName, - logger, - maxArchivedCount = archivedCacheLimit, - ) - - // The syncScope is used to manage coroutines for syncing operations. - // If a coroutineScope is provided, it reuses its context and adds Dispatchers.IO for IO-bound tasks, - // ensuring proper cancellation propagation. If no coroutineScope is provided, a new CoroutineScope - // is created with a SupervisorJob to isolate failures and Dispatchers.IO for efficient IO operations. - private val syncScope = - coroutineScope?.let { - CoroutineScope(it.coroutineContext + Dispatchers.IO) - } ?: CoroutineScope(CoroutineScope(Dispatchers.IO).coroutineContext + SupervisorJob()) - - private var syncStatusJob: Job? = null - private val mutex = Mutex() + /** + * Syncing service for this attachment queue. + * This processes attachment records and performs relevant upload, download, and delete + * operations. + */ + private val syncingService = + SyncingService( + remoteStorage, + localStorage, + attachmentsService, + ::getLocalUri, + errorHandler, + logger, + syncScope, + syncThrottleDuration, + ) + + public var closed: Boolean = false - /** - * Syncing service for this attachment queue. - * This processes attachment records and performs relevant upload, download, and delete - * operations. - */ - private val syncingService = - SyncingService( - remoteStorage, - localStorage, - attachmentsService, - ::getLocalUri, - errorHandler, - logger, - syncScope, - syncThrottleDuration, - ) - - public var closed: Boolean = false + /** + * Initialize the attachment queue by: + * 1. Creating the attachments directory. + * 2. Adding watches for uploads, downloads, and deletes. + * 3. Adding a trigger to run uploads, downloads, and deletes when the device is online after being offline. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun startSync(): Unit = + runWrappedSuspending { + mutex.withLock { + if (closed) { + throw Exception("Attachment queue has been closed") + } - /** - * Initialize the attachment queue by: - * 1. Creating the attachments directory. - * 2. Adding watches for uploads, downloads, and deletes. - * 3. Adding a trigger to run uploads, downloads, and deletes when the device is online after being offline. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun startSync(): Unit = - runWrappedSuspending { - mutex.withLock { - if (closed) { - throw Exception("Attachment queue has been closed") - } + stopSyncingInternal() - stopSyncingInternal() + // Ensure the directory where attachments are downloaded exists. + localStorage.makeDir(attachmentsDirectory) - // Ensure the directory where attachments are downloaded exists. - localStorage.makeDir(attachmentsDirectory) + subdirectories?.forEach { subdirectory -> + localStorage.makeDir(Path(attachmentsDirectory, subdirectory).toString()) + } - subdirectories?.forEach { subdirectory -> - localStorage.makeDir(Path(attachmentsDirectory, subdirectory).toString()) - } + attachmentsService.withContext { context -> + verifyAttachments(context) + } - attachmentsService.withContext { context -> - verifyAttachments(context) - } + syncingService.startSync(syncInterval) - syncingService.startSync(syncInterval) - - // Listen for connectivity changes. - syncStatusJob = - syncScope.launch { - launch { - var previousConnected = db.currentStatus.connected - db.currentStatus.asFlow().collect { status -> - if (!previousConnected && status.connected) { - syncingService.triggerSync() - } - previousConnected = status.connected + // Listen for connectivity changes. + syncStatusJob = + syncScope.launch { + launch { + var previousConnected = db.currentStatus.connected + db.currentStatus.asFlow().collect { status -> + if (!previousConnected && status.connected) { + syncingService.triggerSync() } + previousConnected = status.connected } + } - launch { - // Watch local attachment relationships and sync the attachment records. - watchAttachments().collect { items -> - processWatchedAttachments(items) - } + launch { + // Watch local attachment relationships and sync the attachment records. + watchAttachments().collect { items -> + processWatchedAttachments(items) } } - } + } } + } - /** - * Stops syncing. Syncing may be resumed with [startSync]. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun stopSyncing(): Unit = - mutex.withLock { - stopSyncingInternal() + /** + * Stops syncing. Syncing may be resumed with [startSync]. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun stopSyncing(): Unit = + mutex.withLock { + stopSyncingInternal() + } + + private suspend fun stopSyncingInternal(): Unit = + runWrappedSuspending { + if (closed) { + return@runWrappedSuspending } - private suspend fun stopSyncingInternal(): Unit = - runWrappedSuspending { + syncStatusJob?.cancelAndJoin() + syncStatusJob = null + syncingService.stopSync() + } + + /** + * Closes the queue. + * The queue cannot be used after closing. + * A new queue should be created. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public suspend fun close(): Unit = + runWrappedSuspending { + mutex.withLock { if (closed) { return@runWrappedSuspending } syncStatusJob?.cancelAndJoin() - syncStatusJob = null - syncingService.stopSync() - } - - /** - * Closes the queue. - * The queue cannot be used after closing. - * A new queue should be created. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public suspend fun close(): Unit = - runWrappedSuspending { - mutex.withLock { - if (closed) { - return@runWrappedSuspending - } - - syncStatusJob?.cancelAndJoin() - syncingService.close() - if (coroutineScope == null) { - // Only cancel the internal scope if we created it. - syncScope.coroutineContext[Job]?.takeIf { it.isActive }?.cancelAndJoin() - } - closed = true + syncingService.close() + if (coroutineScope == null) { + // Only cancel the internal scope if we created it. + syncScope.coroutineContext[Job]?.takeIf { it.isActive }?.cancelAndJoin() } + closed = true } + } - /** - * Resolves the filename for new attachment items. - * A new attachment from [watchAttachments] might not include a filename. - * Concatenates the attachment ID and extension by default. - * This method can be overridden for custom behavior. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun resolveNewAttachmentFilename( - attachmentId: String, - fileExtension: String?, - ): String = "$attachmentId.$fileExtension" + /** + * Resolves the filename for new attachment items. + * A new attachment from [watchAttachments] might not include a filename. + * Concatenates the attachment ID and extension by default. + * This method can be overridden for custom behavior. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public open suspend fun resolveNewAttachmentFilename( + attachmentId: String, + fileExtension: String?, + ): String = "$attachmentId.$fileExtension" - /** - * Processes attachment items returned from [watchAttachments]. - * The default implementation asserts the items returned from [watchAttachments] as the definitive - * state for local attachments. - * - * Records currently in the attachment queue which are not present in the items are deleted from - * the queue. - * - * Received items which are not currently in the attachment queue are assumed scheduled for - * download. This requires that locally created attachments should be created with [saveFile] - * before assigning the attachment ID to the relevant watched tables. - * - * This method can be overridden for custom behavior. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun processWatchedAttachments(items: List): Unit = - runWrappedSuspending { + /** + * Processes attachment items returned from [watchAttachments]. + * The default implementation asserts the items returned from [watchAttachments] as the definitive + * state for local attachments. + * + * Records currently in the attachment queue which are not present in the items are deleted from + * the queue. + * + * Received items which are not currently in the attachment queue are assumed scheduled for + * download. This requires that locally created attachments should be created with [saveFile] + * before assigning the attachment ID to the relevant watched tables. + * + * This method can be overridden for custom behavior. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public open suspend fun processWatchedAttachments(items: List): Unit = + runWrappedSuspending { + /** + * Use a lock here to prevent conflicting state updates. + */ + attachmentsService.withContext { attachmentsContext -> /** - * Use a lock here to prevent conflicting state updates. + * Need to get all the attachments which are tracked in the DB. + * We might need to restore an archived attachment. */ - attachmentsService.withContext { attachmentsContext -> - /** - * Need to get all the attachments which are tracked in the DB. - * We might need to restore an archived attachment. - */ - val currentAttachments = attachmentsContext.getAttachments() - val attachmentUpdates = mutableListOf() + val currentAttachments = attachmentsContext.getAttachments() + val attachmentUpdates = mutableListOf() - for (item in items) { - val existingQueueItem = currentAttachments.find { it.id == item.id } + for (item in items) { + val existingQueueItem = currentAttachments.find { it.id == item.id } - if (existingQueueItem == null) { - if (!downloadAttachments) { - continue - } - // This item should be added to the queue. - // This item is assumed to be coming from an upstream sync. - // Locally created new items should be persisted using [saveFile] before - // this point. - val filename = - item.filename ?: resolveNewAttachmentFilename( - attachmentId = item.id, - fileExtension = item.fileExtension, - ) + if (existingQueueItem == null) { + if (!downloadAttachments) { + continue + } + // This item should be added to the queue. + // This item is assumed to be coming from an upstream sync. + // Locally created new items should be persisted using [saveFile] before + // this point. + val filename = + item.filename ?: resolveNewAttachmentFilename( + attachmentId = item.id, + fileExtension = item.fileExtension, + ) + attachmentUpdates.add( + Attachment( + id = item.id, + filename = filename, + state = AttachmentState.QUEUED_DOWNLOAD, + metaData = item.metaData, + ), + ) + } else if + (existingQueueItem.state == AttachmentState.ARCHIVED) { + // The attachment is present again. Need to queue it for sync. + // We might be able to optimize this in future. + if (existingQueueItem.hasSynced == 1) { + // No remote action required, we can restore the record (avoids deletion). + attachmentUpdates.add( + existingQueueItem.copy(state = AttachmentState.SYNCED), + ) + } else { + /** + * The localURI should be set if the record was meant to be downloaded + * and has been synced. If it's missing and hasSynced is false then + * it must be an upload operation. + */ attachmentUpdates.add( - Attachment( - id = item.id, - filename = filename, - state = AttachmentState.QUEUED_DOWNLOAD, - metaData = item.metaData, + existingQueueItem.copy( + state = + if (existingQueueItem.localUri == null) { + AttachmentState.QUEUED_DOWNLOAD + } else { + AttachmentState.QUEUED_UPLOAD + }, ), ) - } else if - (existingQueueItem.state == AttachmentState.ARCHIVED) { - // The attachment is present again. Need to queue it for sync. - // We might be able to optimize this in future. - if (existingQueueItem.hasSynced == 1) { - // No remote action required, we can restore the record (avoids deletion). - attachmentUpdates.add( - existingQueueItem.copy(state = AttachmentState.SYNCED), - ) - } else { - /** - * The localURI should be set if the record was meant to be downloaded - * and has been synced. If it's missing and hasSynced is false then - * it must be an upload operation. - */ - attachmentUpdates.add( - existingQueueItem.copy( - state = - if (existingQueueItem.localUri == null) { - AttachmentState.QUEUED_DOWNLOAD - } else { - AttachmentState.QUEUED_UPLOAD - }, - ), - ) - } } } - - /** - * Archive any items not specified in the watched items except for items pending delete. - */ - currentAttachments - .filter { - it.state != AttachmentState.QUEUED_DELETE && - it.state != AttachmentState.QUEUED_UPLOAD && - null == items.find { update -> update.id == it.id } - }.forEach { - attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED)) - } - - attachmentsContext.saveAttachments(attachmentUpdates) } - } - - /** - * A function which creates a new attachment locally. This new attachment is queued for upload - * after creation. - * - * The filename is resolved using [resolveNewAttachmentFilename]. - * - * A [updateHook] is provided which should be used when assigning relationships to the newly - * created attachment. This hook is executed in the same write transaction which creates the - * attachment record. - * - * This method can be overridden for custom behavior. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun saveFile( - data: Flow, - mediaType: String, - fileExtension: String? = null, - metaData: String? = null, - updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, - ): Attachment = - runWrappedSuspending { - val id = db.get("SELECT uuid() as id") { it.getString("id") } - val filename = - resolveNewAttachmentFilename(attachmentId = id, fileExtension = fileExtension) - val localUri = getLocalUri(filename) - - // Write the file to the filesystem. - val fileSize = localStorage.saveFile(localUri, data) /** - * Starts a write transaction. The attachment record and relevant local relationship - * assignment should happen in the same transaction. + * Archive any items not specified in the watched items except for items pending delete. */ - attachmentsService.withContext { attachmentContext -> - db.writeTransaction { tx -> - val attachment = - Attachment( - id = id, - filename = filename, - size = fileSize, - mediaType = mediaType, - state = AttachmentState.QUEUED_UPLOAD, - localUri = localUri, - metaData = metaData, - ) + currentAttachments + .filter { + it.state != AttachmentState.QUEUED_DELETE && + it.state != AttachmentState.QUEUED_UPLOAD && + null == items.find { update -> update.id == it.id } + }.forEach { + attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED)) + } + + attachmentsContext.saveAttachments(attachmentUpdates) + } + } - /** - * Allow consumers to set relationships to this attachment ID. - */ - updateHook.invoke(tx, attachment) + /** + * A function which creates a new attachment locally. This new attachment is queued for upload + * after creation. + * + * The filename is resolved using [resolveNewAttachmentFilename]. + * + * A [updateHook] is provided which should be used when assigning relationships to the newly + * created attachment. This hook is executed in the same write transaction which creates the + * attachment record. + * + * This method can be overridden for custom behavior. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public open suspend fun saveFile( + data: Flow, + mediaType: String, + fileExtension: String? = null, + metaData: String? = null, + updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, + ): Attachment = + runWrappedSuspending { + val id = db.get("SELECT uuid() as id") { it.getString("id") } + val filename = + resolveNewAttachmentFilename(attachmentId = id, fileExtension = fileExtension) + val localUri = getLocalUri(filename) + + // Write the file to the filesystem. + val fileSize = localStorage.saveFile(localUri, data) - return@writeTransaction attachmentContext.upsertAttachment( - attachment, - tx, + /** + * Starts a write transaction. The attachment record and relevant local relationship + * assignment should happen in the same transaction. + */ + attachmentsService.withContext { attachmentContext -> + db.writeTransaction { tx -> + val attachment = + Attachment( + id = id, + filename = filename, + size = fileSize, + mediaType = mediaType, + state = AttachmentState.QUEUED_UPLOAD, + localUri = localUri, + metaData = metaData, ) - } + + /** + * Allow consumers to set relationships to this attachment ID. + */ + updateHook.invoke(tx, attachment) + + return@writeTransaction attachmentContext.upsertAttachment( + attachment, + tx, + ) } } + } - /** - * A function which creates an attachment delete operation locally. This operation is queued - * for delete. - * The default implementation assumes the attachment record already exists locally. An exception - * is thrown if the record does not exist locally. - * This method can be overridden for custom behavior. - */ - @Throws(PowerSyncException::class, CancellationException::class) - public open suspend fun deleteFile( - attachmentId: String, - updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, - ): Attachment = - runWrappedSuspending { - attachmentsService.withContext { attachmentContext -> - val attachment = - attachmentContext.getAttachment(attachmentId) - ?: throw error("Attachment record with id $attachmentId was not found.") - - db.writeTransaction { tx -> - updateHook.invoke(tx, attachment) - return@writeTransaction attachmentContext.upsertAttachment( - attachment.copy(state = AttachmentState.QUEUED_DELETE), - tx, - ) - } + /** + * A function which creates an attachment delete operation locally. This operation is queued + * for delete. + * The default implementation assumes the attachment record already exists locally. An exception + * is thrown if the record does not exist locally. + * This method can be overridden for custom behavior. + */ + @Throws(PowerSyncException::class, CancellationException::class) + public open suspend fun deleteFile( + attachmentId: String, + updateHook: (context: ConnectionContext, attachment: Attachment) -> Unit, + ): Attachment = + runWrappedSuspending { + attachmentsService.withContext { attachmentContext -> + val attachment = + attachmentContext.getAttachment(attachmentId) + ?: throw error("Attachment record with id $attachmentId was not found.") + + db.writeTransaction { tx -> + updateHook.invoke(tx, attachment) + return@writeTransaction attachmentContext.upsertAttachment( + attachment.copy(state = AttachmentState.QUEUED_DELETE), + tx, + ) } } + } - /** - * Returns the user's storage directory with the attachment path used to load the file. - * Example: filePath: "attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg". - */ - public open fun getLocalUri(filename: String): String = Path(attachmentsDirectory, filename).toString() + /** + * Returns the user's storage directory with the attachment path used to load the file. + * Example: filePath: "attachment-1.jpg" returns "/data/user/0/com.yourdomain.app/files/attachments/attachment-1.jpg". + */ + public open fun getLocalUri(filename: String): String = Path(attachmentsDirectory, filename).toString() - /** - * Removes all archived items. - */ - public suspend fun expireCache() { - var done: Boolean - attachmentsService.withContext { context -> - do { - done = syncingService.deleteArchivedAttachments(context) - } while (!done) - } + /** + * Removes all archived items. + */ + public suspend fun expireCache() { + var done: Boolean + attachmentsService.withContext { context -> + do { + done = syncingService.deleteArchivedAttachments(context) + } while (!done) } + } - /** - * Clears the attachment queue and deletes all attachment files. - */ - public suspend fun clearQueue() { - attachmentsService.withContext { - it.clearQueue() - } - // Remove the attachments directory. - localStorage.rmDir(attachmentsDirectory) + /** + * Clears the attachment queue and deletes all attachment files. + */ + public suspend fun clearQueue() { + attachmentsService.withContext { + it.clearQueue() } + // Remove the attachments directory. + localStorage.rmDir(attachmentsDirectory) + } - /** - * Cleans up stale attachments. - */ - private suspend fun verifyAttachments(context: AttachmentContext) { - val attachments = context.getActiveAttachments() - val updates = mutableListOf() + /** + * Cleans up stale attachments. + */ + private suspend fun verifyAttachments(context: AttachmentContext) { + val attachments = context.getActiveAttachments() + val updates = mutableListOf() - for (attachment in attachments) { - if (attachment.localUri == null) { - continue - } - val exists = localStorage.fileExists(attachment.localUri) - if ( - attachment.state == AttachmentState.SYNCED || attachment.state == AttachmentState.QUEUED_UPLOAD && !exists - ) { - updates.add( - attachment.copy( - state = AttachmentState.ARCHIVED, - localUri = null, - ), - ) - } + for (attachment in attachments) { + if (attachment.localUri == null) { + continue + } + val exists = localStorage.fileExists(attachment.localUri) + if ( + attachment.state == AttachmentState.SYNCED || attachment.state == AttachmentState.QUEUED_UPLOAD && !exists + ) { + updates.add( + attachment.copy( + state = AttachmentState.ARCHIVED, + localUri = null, + ), + ) } - - context.saveAttachments(updates) } + + context.saveAttachments(updates) } +} diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt index a013111f..fdb3dc3f 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt @@ -11,6 +11,13 @@ import kotlinx.datetime.Clock import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +/** + * Default implementation of [AttachmentContext]. + * + * This class provides the standard logic for managing attachments in a SQLite table. + * Users can override this class if they need custom logic for handling columns or other + * database operations related to attachments. + */ public open class AttachmentContextImpl( public val db: PowerSyncDatabase, public val table: String, diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt index c94c09eb..9755d026 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentServiceImpl.kt @@ -12,7 +12,10 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock /** - * Service for interacting with the local attachment records. + * Default implementation of [AttachmentService]. + * + * This class provides the standard logic for interacting with local attachment records. + * Users can extend this class to customize the behavior for their specific use cases. */ public open class AttachmentServiceImpl( private val db: PowerSyncDatabase, diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt index e194be2f..ea9799fa 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/storage/IOLocalStorageAdapter.kt @@ -19,9 +19,9 @@ import kotlinx.io.readByteArray /** * Storage adapter for local storage using the KotlinX IO library */ -public class IOLocalStorageAdapter : LocalStorage { - private val fileSystem: FileSystem = SystemFileSystem - +public open class IOLocalStorageAdapter( + private val fileSystem: FileSystem = SystemFileSystem, +) : LocalStorage { public override suspend fun saveFile( filePath: String, data: Flow, diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index cd9af5a7..a7411ada 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -12,7 +12,6 @@ import com.powersync.attachments.SyncErrorHandler import com.powersync.utils.throttle import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Job import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.channels.BufferOverflow @@ -34,6 +33,9 @@ import kotlin.time.Duration.Companion.seconds * periodically syncing attachment states. It ensures proper lifecycle management * of sync operations and provides mechanisms for error handling and retries. * + * The class provides a default implementation for syncing operations, which can be + * overridden by subclasses to customize behavior as needed. + * * @property remoteStorage The remote storage implementation for handling file operations. * @property localStorage The local storage implementation for managing files locally. * @property attachmentsService The service for managing attachment states and operations. @@ -43,269 +45,267 @@ import kotlin.time.Duration.Companion.seconds * @property syncThrottle The minimum duration between consecutive sync operations. * @property scope The coroutine scope used for managing sync operations. */ -public class SyncingService - @OptIn(DelicateCoroutinesApi::class) - constructor( - private val remoteStorage: RemoteStorage, - private val localStorage: LocalStorage, - private val attachmentsService: AttachmentService, - private val getLocalUri: suspend (String) -> String, - private val errorHandler: SyncErrorHandler?, - private val logger: Logger, - private val scope: CoroutineScope, - private val syncThrottle: Duration = 5.seconds, - ) { - private val mutex = Mutex() - private var syncJob: Job? = null +public open class SyncingService( + private val remoteStorage: RemoteStorage, + private val localStorage: LocalStorage, + private val attachmentsService: AttachmentService, + private val getLocalUri: suspend (String) -> String, + private val errorHandler: SyncErrorHandler?, + private val logger: Logger, + private val scope: CoroutineScope, + private val syncThrottle: Duration = 5.seconds, +) { + private val mutex = Mutex() + private var syncJob: Job? = null - /** - * Used to trigger the sync process either manually or periodically - */ - private val syncTriggerFlow = MutableSharedFlow(replay = 0) + /** + * Used to trigger the sync process either manually or periodically + */ + private val syncTriggerFlow = MutableSharedFlow(replay = 0) - /** - * Starts the syncing process, including periodic and event-driven sync operations. - * - * @param period The interval at which periodic sync operations are triggered. - */ - public suspend fun startSync(period: Duration = 30.seconds): Unit = - mutex.withLock { - syncJob?.cancelAndJoin() + /** + * Starts the syncing process, including periodic and event-driven sync operations. + * + * @param period The interval at which periodic sync operations are triggered. + */ + public suspend fun startSync(period: Duration = 30.seconds): Unit = + mutex.withLock { + syncJob?.cancelAndJoin() - syncJob = - scope.launch { - launch { - merge( - // Handles manual triggers for sync events - syncTriggerFlow.asSharedFlow(), - // Triggers the sync process whenever an underlaying change to the - // attachments table happens - attachmentsService - .watchActiveAttachments(), - ) - // We only use these flows to trigger the process. We can skip multiple invocations - // while we are processing. We will always process on the trailing edge. - // This buffer operation should automatically be applied to all merged sources. - .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) - .throttle(syncThrottle) - .collect { - attachmentsService.withContext { context -> - /** - * Gets and performs the operations for active attachments which are - * pending download, upload, or delete. - */ - try { - val attachments = context.getActiveAttachments() - // Performs pending operations and updates attachment states - handleSync(attachments, context) + syncJob = + scope.launch { + launch { + merge( + // Handles manual triggers for sync events + syncTriggerFlow.asSharedFlow(), + // Triggers the sync process whenever an underlaying change to the + // attachments table happens + attachmentsService + .watchActiveAttachments(), + ) + // We only use these flows to trigger the process. We can skip multiple invocations + // while we are processing. We will always process on the trailing edge. + // This buffer operation should automatically be applied to all merged sources. + .buffer(1, onBufferOverflow = BufferOverflow.DROP_OLDEST) + .throttle(syncThrottle) + .collect { + attachmentsService.withContext { context -> + /** + * Gets and performs the operations for active attachments which are + * pending download, upload, or delete. + */ + try { + val attachments = context.getActiveAttachments() + // Performs pending operations and updates attachment states + handleSync(attachments, context) - // Cleanup archived attachments - deleteArchivedAttachments(context) - } catch (ex: Exception) { - if (ex is CancellationException) { - throw ex - } - // Rare exceptions caught here will be swallowed and retried on the - // next tick. - logger.e("Caught exception when processing attachments $ex") + // Cleanup archived attachments + deleteArchivedAttachments(context) + } catch (ex: Exception) { + if (ex is CancellationException) { + throw ex } + // Rare exceptions caught here will be swallowed and retried on the + // next tick. + logger.e("Caught exception when processing attachments $ex") } } - } - - launch { - logger.i("Periodically syncing attachments") - while (true) { - syncTriggerFlow.emit(Unit) - delay(period) } - } } - } - /** - * Enqueues a sync operation - */ - public suspend fun triggerSync() { - syncTriggerFlow.emit(Unit) + launch { + logger.i("Periodically syncing attachments") + while (true) { + syncTriggerFlow.emit(Unit) + delay(period) + } + } + } } - /** - * Stops all ongoing sync operations. - */ - public suspend fun stopSync(): Unit = - mutex.withLock { - syncJob?.cancelAndJoin() - } + /** + * Enqueues a sync operation + */ + public suspend fun triggerSync() { + syncTriggerFlow.emit(Unit) + } - /** - * Closes the syncing service, stopping all operations and releasing resources. - */ - public suspend fun close() { - stopSync() + /** + * Stops all ongoing sync operations. + */ + public suspend fun stopSync(): Unit = + mutex.withLock { + syncJob?.cancelAndJoin() } - /** - * Handles syncing operations for a list of attachments, including downloading, - * uploading, and deleting files based on their states. - * - * @param attachments The list of attachments to process. - * @param context The attachment context used for managing attachment states. - */ - private suspend fun handleSync( - attachments: List, - context: AttachmentContext, - ) { - val updatedAttachments = mutableListOf() - try { - for (attachment in attachments) { - when (attachment.state) { - AttachmentState.QUEUED_DOWNLOAD -> { - logger.i("Downloading ${attachment.filename}") - updatedAttachments.add(downloadAttachment(attachment)) - } + /** + * Closes the syncing service, stopping all operations and releasing resources. + */ + public suspend fun close() { + stopSync() + } - AttachmentState.QUEUED_UPLOAD -> { - logger.i("Uploading ${attachment.filename}") - updatedAttachments.add(uploadAttachment(attachment)) - } + /** + * Handles syncing operations for a list of attachments, including downloading, + * uploading, and deleting files based on their states. + * + * @param attachments The list of attachments to process. + * @param context The attachment context used for managing attachment states. + */ + private suspend fun handleSync( + attachments: List, + context: AttachmentContext, + ) { + val updatedAttachments = mutableListOf() + try { + for (attachment in attachments) { + when (attachment.state) { + AttachmentState.QUEUED_DOWNLOAD -> { + logger.i("Downloading ${attachment.filename}") + updatedAttachments.add(downloadAttachment(attachment)) + } - AttachmentState.QUEUED_DELETE -> { - logger.i("Deleting ${attachment.filename}") - updatedAttachments.add(deleteAttachment(attachment)) - } + AttachmentState.QUEUED_UPLOAD -> { + logger.i("Uploading ${attachment.filename}") + updatedAttachments.add(uploadAttachment(attachment)) + } - AttachmentState.SYNCED -> {} - AttachmentState.ARCHIVED -> {} + AttachmentState.QUEUED_DELETE -> { + logger.i("Deleting ${attachment.filename}") + updatedAttachments.add(deleteAttachment(attachment)) } - } - // Update the state of processed attachments - context.saveAttachments(updatedAttachments) - } catch (error: Exception) { - // We retry, on the next invocation, whenever there are errors on this level - logger.e("Error during sync: ${error.message}") + AttachmentState.SYNCED -> {} + AttachmentState.ARCHIVED -> {} + } } - } - /** - * Uploads an attachment from local storage to remote storage. - * - * @param attachment The attachment to upload. - * @return The updated attachment with its new state. - */ - private suspend fun uploadAttachment(attachment: Attachment): Attachment { - try { - if (attachment.localUri == null) { - throw PowerSyncException( - "No localUri for attachment $attachment", - cause = Exception("attachment.localUri == null"), - ) - } + // Update the state of processed attachments + context.saveAttachments(updatedAttachments) + } catch (error: Exception) { + // We retry, on the next invocation, whenever there are errors on this level + logger.e("Error during sync: ${error.message}") + } + } - remoteStorage.uploadFile( - localStorage.readFile(attachment.localUri), - attachment, + /** + * Uploads an attachment from local storage to remote storage. + * + * @param attachment The attachment to upload. + * @return The updated attachment with its new state. + */ + private suspend fun uploadAttachment(attachment: Attachment): Attachment { + try { + if (attachment.localUri == null) { + throw PowerSyncException( + "No localUri for attachment $attachment", + cause = Exception("attachment.localUri == null"), ) - logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") - return attachment.copy(state = AttachmentState.SYNCED, hasSynced = 1) - } catch (e: Exception) { - logger.e("Upload attachment error for attachment $attachment: ${e.message}") - if (errorHandler != null) { - val shouldRetry = errorHandler.onUploadError(attachment, e) - if (!shouldRetry) { - logger.i("Attachment with ID ${attachment.id} has been archived") - return attachment.copy(state = AttachmentState.ARCHIVED) - } - } + } - // Retry the upload (same state) - return attachment + remoteStorage.uploadFile( + localStorage.readFile(attachment.localUri), + attachment, + ) + logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") + return attachment.copy(state = AttachmentState.SYNCED, hasSynced = 1) + } catch (e: Exception) { + logger.e("Upload attachment error for attachment $attachment: ${e.message}") + if (errorHandler != null) { + val shouldRetry = errorHandler.onUploadError(attachment, e) + if (!shouldRetry) { + logger.i("Attachment with ID ${attachment.id} has been archived") + return attachment.copy(state = AttachmentState.ARCHIVED) + } } + + // Retry the upload (same state) + return attachment } + } + /** + * Downloads an attachment from remote storage and saves it to local storage. + * + * @param attachment The attachment to download. + * @return The updated attachment with its new state. + */ + private suspend fun downloadAttachment(attachment: Attachment): Attachment { /** - * Downloads an attachment from remote storage and saves it to local storage. - * - * @param attachment The attachment to download. - * @return The updated attachment with its new state. + * When downloading an attachment we take the filename and resolve + * the local_uri where the file will be stored */ - private suspend fun downloadAttachment(attachment: Attachment): Attachment { - /** - * When downloading an attachment we take the filename and resolve - * the local_uri where the file will be stored - */ - val attachmentPath = getLocalUri(attachment.filename) + val attachmentPath = getLocalUri(attachment.filename) - try { - val fileFlow = remoteStorage.downloadFile(attachment) - localStorage.saveFile(attachmentPath, fileFlow) - logger.i("Downloaded file \"${attachment.id}\"") + try { + val fileFlow = remoteStorage.downloadFile(attachment) + localStorage.saveFile(attachmentPath, fileFlow) + logger.i("Downloaded file \"${attachment.id}\"") - // The attachment has been downloaded locally - return attachment.copy( - localUri = attachmentPath, - state = AttachmentState.SYNCED, - hasSynced = 1, - ) - } catch (e: Exception) { - if (errorHandler != null) { - val shouldRetry = errorHandler.onDownloadError(attachment, e) - if (!shouldRetry) { - logger.i("Attachment with ID ${attachment.id} has been archived") - return attachment.copy(state = AttachmentState.ARCHIVED) - } + // The attachment has been downloaded locally + return attachment.copy( + localUri = attachmentPath, + state = AttachmentState.SYNCED, + hasSynced = 1, + ) + } catch (e: Exception) { + if (errorHandler != null) { + val shouldRetry = errorHandler.onDownloadError(attachment, e) + if (!shouldRetry) { + logger.i("Attachment with ID ${attachment.id} has been archived") + return attachment.copy(state = AttachmentState.ARCHIVED) } - - logger.e("Download attachment error for attachment $attachment: ${e.message}") - // Return the same state, this will cause a retry - return attachment } + + logger.e("Download attachment error for attachment $attachment: ${e.message}") + // Return the same state, this will cause a retry + return attachment } + } - /** - * Deletes an attachment from remote and local storage, and removes it from the queue. - * - * @param attachment The attachment to delete. - * @return The updated attachment with its new state. - */ - private suspend fun deleteAttachment(attachment: Attachment): Attachment { - try { - remoteStorage.deleteFile(attachment) - if (attachment.localUri != null && localStorage.fileExists(attachment.localUri)) { - localStorage.deleteFile(attachment.localUri) - } - return attachment.copy(state = AttachmentState.ARCHIVED) - } catch (e: Exception) { - if (errorHandler != null) { - val shouldRetry = errorHandler.onDeleteError(attachment, e) - if (!shouldRetry) { - logger.i("Attachment with ID ${attachment.id} has been archived") - return attachment.copy(state = AttachmentState.ARCHIVED) - } + /** + * Deletes an attachment from remote and local storage, and removes it from the queue. + * + * @param attachment The attachment to delete. + * @return The updated attachment with its new state. + */ + private suspend fun deleteAttachment(attachment: Attachment): Attachment { + try { + remoteStorage.deleteFile(attachment) + if (attachment.localUri != null && localStorage.fileExists(attachment.localUri)) { + localStorage.deleteFile(attachment.localUri) + } + return attachment.copy(state = AttachmentState.ARCHIVED) + } catch (e: Exception) { + if (errorHandler != null) { + val shouldRetry = errorHandler.onDeleteError(attachment, e) + if (!shouldRetry) { + logger.i("Attachment with ID ${attachment.id} has been archived") + return attachment.copy(state = AttachmentState.ARCHIVED) } - // We'll retry this - logger.e("Error deleting attachment: ${e.message}") - return attachment } + // We'll retry this + logger.e("Error deleting attachment: ${e.message}") + return attachment } + } - /** - * Deletes archived attachments from local storage. - * - * @param context The attachment context used to retrieve and manage archived attachments. - * @return `true` if all archived attachments were successfully deleted, `false` otherwise. - */ - public suspend fun deleteArchivedAttachments(context: AttachmentContext): Boolean = - context.deleteArchivedAttachments { pendingDelete -> - for (attachment in pendingDelete) { - if (attachment.localUri == null) { - continue - } - if (!localStorage.fileExists(attachment.localUri)) { - continue - } - localStorage.deleteFile(attachment.localUri) + /** + * Deletes archived attachments from local storage. + * + * @param context The attachment context used to retrieve and manage archived attachments. + * @return `true` if all archived attachments were successfully deleted, `false` otherwise. + */ + public suspend fun deleteArchivedAttachments(context: AttachmentContext): Boolean = + context.deleteArchivedAttachments { pendingDelete -> + for (attachment in pendingDelete) { + if (attachment.localUri == null) { + continue + } + if (!localStorage.fileExists(attachment.localUri)) { + continue } + localStorage.deleteFile(attachment.localUri) } - } + } +} diff --git a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt index 70120158..1d163cbf 100644 --- a/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt +++ b/demos/android-supabase-todolist/app/src/main/java/com/powersync/androidexample/App.kt @@ -12,6 +12,7 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.rememberUpdatedState import androidx.compose.ui.Modifier import com.powersync.PowerSyncDatabase +import com.powersync.androidexample.BuildConfig import com.powersync.androidexample.ui.CameraService import com.powersync.attachments.AttachmentQueue import com.powersync.attachments.WatchedAttachmentItem @@ -29,7 +30,6 @@ import com.powersync.demos.screens.SignInScreen import com.powersync.demos.screens.SignUpScreen import com.powersync.demos.screens.TodosScreen import kotlinx.coroutines.runBlocking -import com.powersync.androidexample.BuildConfig @Composable fun App( @@ -63,8 +63,8 @@ fun App( id = it.getString("photo_id"), fileExtension = "jpg", ) - } } + }, ) } else { null diff --git a/demos/android-supabase-todolist/local.properties.example b/demos/android-supabase-todolist/local.properties.example index a1ac1145..1acbc028 100644 --- a/demos/android-supabase-todolist/local.properties.example +++ b/demos/android-supabase-todolist/local.properties.example @@ -11,7 +11,8 @@ sdk.dir=/Users/dominic/Library/Android/sdk SUPABASE_URL=https://foo.supabase.co SUPABASE_ANON_KEY=foo -SUPABASE_STORAGE_BUCKET=media # optional attachment bucket, set to null to disable +# optional attachment bucket, set to null to disable +SUPABASE_STORAGE_BUCKET=null POWERSYNC_URL=https://foo.powersync.journeyapps.com # Set to true to use released PowerSync packages instead of the ones built locally. USE_RELEASED_POWERSYNC_VERSIONS=false From 7cf5638222d9d595021a9bd7c2a6e9e1ff7c9961 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 22 Apr 2025 11:46:22 +0200 Subject: [PATCH 51/51] Improve boolean hasSynced. Make tests more stable. --- .../kotlin/com/powersync/AttachmentsTest.kt | 11 +++---- .../com/powersync/attachments/Attachment.kt | 4 +-- .../powersync/attachments/AttachmentQueue.kt | 29 ++++++++++++++----- .../implementation/AttachmentContextImpl.kt | 2 +- .../attachments/sync/SyncingService.kt | 4 +-- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt index 151e8560..db05a89b 100644 --- a/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/AttachmentsTest.kt @@ -275,14 +275,12 @@ class AttachmentsTest { """, ) - var nextRecord: Attachment? = attachmentQuery.awaitItem().first() - if (nextRecord?.state == AttachmentState.ARCHIVED) { - nextRecord = attachmentQuery.awaitItem().getOrNull(0) + waitFor { + var nextRecord: Attachment? = attachmentQuery.awaitItem().firstOrNull() + // The record should have been deleted + nextRecord shouldBe null } - // The record should have been deleted - nextRecord shouldBe null - // The file should have been deleted from storage queue.localStorage.fileExists(localUri) shouldBe false @@ -522,7 +520,6 @@ class AttachmentsTest { remoteStorage = remote, attachmentsDirectory = getAttachmentsDir(), watchAttachments = { watchAttachments(database) }, - archivedCacheLimit = 0, errorHandler = object : SyncErrorHandler { override suspend fun onDownloadError( diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt index 377055e9..4c12be18 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/Attachment.kt @@ -72,7 +72,7 @@ public data class Attachment( val localUri: String? = null, val mediaType: String? = null, val size: Long? = null, - val hasSynced: Int = 0, + val hasSynced: Boolean = false, val metaData: String? = null, ) { public companion object { @@ -91,7 +91,7 @@ public data class Attachment( mediaType = cursor.getStringOptional(name = "media_type"), size = cursor.getLongOptional("size"), state = AttachmentState.fromLong(cursor.getLong("state")), - hasSynced = cursor.getLong("has_synced").toInt(), + hasSynced = cursor.getLong("has_synced").toInt() > 0, metaData = cursor.getStringOptional("meta_data"), ) } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt index 2705159b..45e5eade 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/AttachmentQueue.kt @@ -363,7 +363,7 @@ public open class AttachmentQueue( (existingQueueItem.state == AttachmentState.ARCHIVED) { // The attachment is present again. Need to queue it for sync. // We might be able to optimize this in future. - if (existingQueueItem.hasSynced == 1) { + if (existingQueueItem.hasSynced) { // No remote action required, we can restore the record (avoids deletion). attachmentUpdates.add( existingQueueItem.copy(state = AttachmentState.SYNCED), @@ -389,13 +389,25 @@ public open class AttachmentQueue( } /** - * Archive any items not specified in the watched items except for items pending delete. + * Archive any items not specified in the watched items. + * For QUEUED_DELETE or QUEUED_UPLOAD states, archive only if hasSynced is true. + * For other states, archive if the record is not found in the items. */ currentAttachments - .filter { - it.state != AttachmentState.QUEUED_DELETE && - it.state != AttachmentState.QUEUED_UPLOAD && - null == items.find { update -> update.id == it.id } + .filter { attachment -> + val notInWatchedItems = + items.find { update -> update.id == attachment.id } == null + if (notInWatchedItems) { + when (attachment.state) { + // Archive these record if they have synced + AttachmentState.QUEUED_DELETE, AttachmentState.QUEUED_UPLOAD -> attachment.hasSynced + // Other states, such as QUEUED_DOWNLOAD can be archived if they are not present in watched items + else -> true + } + } else { + // The record is present in watched items, no need to archive it + false + } }.forEach { attachmentUpdates.add(it.copy(state = AttachmentState.ARCHIVED)) } @@ -484,7 +496,10 @@ public open class AttachmentQueue( db.writeTransaction { tx -> updateHook.invoke(tx, attachment) return@writeTransaction attachmentContext.upsertAttachment( - attachment.copy(state = AttachmentState.QUEUED_DELETE), + attachment.copy( + state = AttachmentState.QUEUED_DELETE, + hasSynced = false, + ), tx, ) } diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt index fdb3dc3f..de64fbad 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/implementation/AttachmentContextImpl.kt @@ -191,7 +191,7 @@ public open class AttachmentContextImpl( updatedRecord.mediaType, updatedRecord.size, updatedRecord.state.ordinal, - updatedRecord.hasSynced, + if (updatedRecord.hasSynced) 1 else 0, updatedRecord.metaData, ), ) diff --git a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt index a7411ada..5678d626 100644 --- a/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt +++ b/core/src/commonMain/kotlin/com/powersync/attachments/sync/SyncingService.kt @@ -208,7 +208,7 @@ public open class SyncingService( attachment, ) logger.i("Uploaded attachment \"${attachment.id}\" to Cloud Storage") - return attachment.copy(state = AttachmentState.SYNCED, hasSynced = 1) + return attachment.copy(state = AttachmentState.SYNCED, hasSynced = true) } catch (e: Exception) { logger.e("Upload attachment error for attachment $attachment: ${e.message}") if (errorHandler != null) { @@ -246,7 +246,7 @@ public open class SyncingService( return attachment.copy( localUri = attachmentPath, state = AttachmentState.SYNCED, - hasSynced = 1, + hasSynced = true, ) } catch (e: Exception) { if (errorHandler != null) {