Update core extension, download binaries (#303)

Author: simolus3

.github/workflows/prebuild_assets.yml

@@ -61,7 +61,7 @@ jobs:
       - name: Compile SQLite with Gradle
         if: steps.cache_prebuild.outputs.cache-hit != 'true'
         run: |
-          ./gradlew --scan internal:prebuild-binaries:compileAll
+          ./gradlew --scan --configure-on-demand internal:prebuild-binaries:compileAll
         shell: bash
       - uses: actions/upload-artifact@v5
         id: upload

.github/workflows/test.yml

@@ -138,4 +138,4 @@ jobs:
           disable-animations: false
           avd-name: $AVD_NAME
           emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
-          script: ./gradlew --scan core-tests-android:connectedCheck
+          script: ./gradlew --scan --configure-on-demand core-tests-android:connectedCheck

common/build.gradle.kts

@@ -31,43 +31,46 @@ val downloadPowersyncDesktopBinaries by tasks.registering(Download::class) {
     val coreVersion =
         libs.versions.powersync.core
             .get()
-    val linux_aarch64 =
-        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/libpowersync_aarch64.linux.so"
-    val linux_x64 =
-        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/libpowersync_x64.linux.so"
-    val macos_aarch64 =
-        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/libpowersync_aarch64.macos.dylib"
-    val macos_x64 =
-        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/libpowersync_x64.macos.dylib"
-    val windows_x64 =
-        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/powersync_x64.dll"
+
+    fun downloadUrl(filename: String): String {
+        return "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/$filename"
+    }
+
+    val linux_aarch64 = "libpowersync_aarch64.linux.so"
+    val linux_x64 = "libpowersync_x64.linux.so"
+    val macos_aarch64 = "libpowersync_aarch64.macos.dylib"
+    val macos_x64 = "libpowersync_x64.macos.dylib"
+    val windows_aarch64 = "powersync_aarch64.dll"
+    val windows_x64 = "powersync_x64.dll"
 
     val includeAllPlatformsForJvmBuild =
         project.findProperty("powersync.binaries.allPlatforms") == "true"
     val os = OperatingSystem.current()
 
     // The jar we're releasing for JVM clients needs to include the core extension. For local tests, it's enough to only
     // download the extension for the OS running the build. For releases, we want to include them all.
-    // We're not compiling native code for JVM builds here (we're doing that for Android only), so we just have to
-    // fetch prebuilt binaries from the powersync-sqlite-core repository.
+    // We're not compiling native code for JVM builds here, so we just have to fetch prebuilt binaries from the
+    // powersync-sqlite-core repository.
     if (includeAllPlatformsForJvmBuild) {
-        src(listOf(linux_aarch64, linux_x64, macos_aarch64, macos_x64, windows_x64))
+        val allSources = sequenceOf(linux_aarch64, linux_x64, macos_aarch64, macos_x64, windows_aarch64, windows_x64)
+            .map(::downloadUrl)
+            .toList()
+        src(allSources)
     } else {
         val (aarch64, x64) =
             when {
                 os.isLinux -> linux_aarch64 to linux_x64
                 os.isMacOsX -> macos_aarch64 to macos_x64
-                os.isWindows -> null to windows_x64
+                os.isWindows -> windows_aarch64 to windows_x64
                 else -> error("Unknown operating system: $os")
             }
         val arch = System.getProperty("os.arch")
-        src(
-            when (arch) {
-                "aarch64" -> listOfNotNull(aarch64)
-                "amd64", "x86_64" -> listOfNotNull(x64)
-                else -> error("Unsupported architecture: $arch")
-            },
-        )
+        val filename = when (arch) {
+            "aarch64" -> aarch64
+            "amd64", "x86_64" -> x64
+            else -> error("Unsupported architecture: $arch")
+        }
+        src(downloadUrl(filename))
     }
     dest(binariesFolder.map { it.dir("powersync") })
     onlyIfModified(true)

common/src/commonMain/kotlin/com/powersync/db/internal/PowerSyncVersion.kt

@@ -18,7 +18,7 @@ internal data class PowerSyncVersion(
     override fun toString(): String = "$major.$minor.$patch"
 
     companion object {
-        val MINIMUM: PowerSyncVersion = PowerSyncVersion(0, 4, 0)
+        val MINIMUM: PowerSyncVersion = PowerSyncVersion(0, 4, 10)
 
         fun parse(from: String): PowerSyncVersion {
             val versionInts: List<Int> =

gradle/libs.versions.toml

@@ -18,7 +18,7 @@ serialization = "1.9.0"
 kotlinx-io = "0.8.0"
 ktor = "3.3.3"
 uuid = "0.8.4"
-powersync-core = "0.4.9"
+powersync-core = "0.4.10"
 rsocket = "0.20.0"
 turbine = "1.2.1"
 kotest = "5.9.1" # we can't upgrade to 6.x because that requires Java 11 or above (we need Java 8 support)

internal/prebuild-binaries/build.gradle.kts

@@ -10,9 +10,12 @@ import com.powersync.compile.JniTarget
 import kotlin.io.path.absolutePathString
 
 plugins {
+    base // For the clean task
     alias(libs.plugins.downloadPlugin)
 }
 
+val lastKotlinSdkRelease = project.property("LIBRARY_VERSION") as String
+
 val sqlite3McVersion = "2.2.6"
 val sqlite3BaseVersion = "3.51.1"
 val sqlite3ReleaseYear = "2025"
@@ -41,6 +44,14 @@ val downloadSqlite3MultipleCipherSources by tasks.registering(Download::class) {
     overwrite(false)
 }
 
+val downloadPrebuiltsFromRelease by tasks.registering(Download::class) {
+    val zipFileName = "prebuilts-$lastKotlinSdkRelease.zip"
+    src("https://github.com/powersync-ja/powersync-kotlin/releases/download/v$lastKotlinSdkRelease/prebuilt_libraries.zip")
+    dest(layout.buildDirectory.dir("downloads").map { it.file(zipFileName) })
+    onlyIfNewer(true)
+    overwrite(false)
+}
+
 val unzipSQLiteSources by tasks.registering(UnzipSqlite::class) {
     val zip = downloadSQLiteSources.map { it.outputs.files.singleFile }
     inputs.file(zip)
@@ -62,6 +73,33 @@ val unzipSqlite3MultipleCipherSources by tasks.registering(UnzipSqlite::class) {
     )
 }
 
+abstract class ExtractPrebuilts: Copy() {
+    @get:OutputDirectory
+    abstract val outputDir: DirectoryProperty
+
+    init {
+        into(outputDir)
+    }
+}
+
+val unzipPrebuiltsFromLastRelease by tasks.registering(ExtractPrebuilts::class) {
+    val zip = downloadPrebuiltsFromRelease.map { it.outputs.files.singleFile }
+
+    from(zipTree(zip)) {
+        include("internal/prebuild-binaries/build/output/**")
+
+        eachFile {
+            relativePath = RelativePath(
+                relativePath.isFile,
+                // Remove segments: [internal, prebuild-binaries, build, output]
+                *relativePath.segments.drop(4).toTypedArray()
+            )
+        }
+    }
+
+    outputDir.set(layout.buildDirectory.dir("downloads/from_last_release"))
+}
+
 val prepareAndroidBuild by tasks.registering(Copy::class) {
     from(unzipSqlite3MultipleCipherSources.flatMap { it.destination }) {
         include(
@@ -186,7 +224,6 @@ val compileAll by tasks.registering {
     dependsOn(compileJni)
 }
 
-val hasPrebuiltAssets = providers.gradleProperty("hasPrebuiltAssets").map { it.toBooleanStrict() }
 
 val nativeSqliteConfiguration by configurations.creating {
     isCanBeResolved = false
@@ -202,12 +239,33 @@ val androidBuildSourceConfiguration by configurations.creating {
 }
 
 artifacts {
+    val hasPrebuiltAssets = providers.gradleProperty("hasPrebuiltAssets").map { it.toBooleanStrict() }
+    val isInCI = providers.environmentVariable("CI").isPresent
+
+    // There are three ways to obtain the static and JNI libraries:
+    //
+    //  1. We can download and compile from source. This requires special toolchain dependencies (see readme) and is
+    //     slow.
+    //  2. We can download compiled binaries from the last GitHub release. We don't want to do this in CI to ensure
+    //     updates to the build are reflected in tests. It's a good fit for builds on developer machines though.
+    //  3. In CI, we run 1 in a separate step and re-use its outputs across multiple builds. -PhasPrebuiltAssets=true is
+    //     passed for consumers of those artifacts.
     if (hasPrebuiltAssets.getOrElse(false)) {
-        // In CI builds, we set hasPrebuiltAssets=true. In that case, contents of build/output have been downloaded from
-        // cache and don't need to be rebuilt.
+        // Case 3: Re-use downloaded assets.
         add(nativeSqliteConfiguration.name, layout.buildDirectory.dir("output/static"))
         add(jniSqlite3McConfiguration.name, layout.buildDirectory.dir("output/jni"))
+    } else if (!isInCI) {
+        // Developer machine, case 2. Download from last release.
+        add(
+            nativeSqliteConfiguration.name,
+            unzipPrebuiltsFromLastRelease.flatMap { it.outputDir.dir("static") }
+        )
+        add(
+            jniSqlite3McConfiguration.name,
+            unzipPrebuiltsFromLastRelease.flatMap { it.outputDir.dir("jni") }
+        )
     } else {
+        // Case 1, compile from source.
         add(nativeSqliteConfiguration.name, compileNative)
         add(jniSqlite3McConfiguration.name, compileJni)
     }