From e459274ea0cff6323c836ffc6b3b3650e1325282 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Mon, 8 Sep 2025 23:18:05 -0400 Subject: [PATCH 1/9] dataconnect: replace `missingversions.py` with the `:firebase-dataconnet:updateJson` gradle task --- .github/workflows/dataconnect.yml | 49 +++ .../connectors/connectors.gradle.kts | 96 ----- .../firebase-dataconnect.gradle.kts | 15 + .../gradleplugin/plugin/build.gradle.kts | 2 + .../DataConnectExecutableDownloadTask.kt | 2 +- .../DataConnectExecutableVersionRegistry.kt | 35 +- .../gradle/plugin/DataConnectProviders.kt | 2 +- ...UpdateDataConnectExecutableVersionsTask.kt | 328 ++++++++++++------ .../scripts/missingversions.py | 71 ---- .../scripts/update_versions_json.sh | 32 ++ 10 files changed, 334 insertions(+), 298 deletions(-) delete mode 100755 firebase-dataconnect/scripts/missingversions.py create mode 100755 firebase-dataconnect/scripts/update_versions_json.sh diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index aef25709b06..b2d98b214de 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -347,6 +347,53 @@ jobs: working-directory: firebase-dataconnect/ci run: pyright --warnings --stats + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + show-progress: false + + emulator-versions-outdated-check: + continue-on-error: false + runs-on: ubuntu-latest + steps: + - uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0 + with: + java-version: ${{ env.FDC_JAVA_VERSION }} + distribution: temurin + - uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8 + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }} + - uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 + - name: Gradle updateJson + run: | + set -euo pipefail + set -v + + # Speed up build times and also avoid configuring firebase-crashlytics-ndk + # which is finicky integrating with the Android NDK. + echo >> gradle.properties + echo "org.gradle.configureondemand=true" >> gradle.properties + + readonly json_file='firebase-dataconnect/gradleplugin/plugin/src/main/resources/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersions.json' + cp "${json_file}" '${{ runner.temp }}/DataConnectExecutableVersions.before.json' + + ./gradlew \ + --warning-mode all \ + ${{ (inputs.gradleInfoLog && '--info') || '' }} \ + :firebase-dataconnect:updateJson + + cp "${json_file}" '${{ runner.temp }}/DataConnectExecutableVersions.after.json' + + if diff DataConnectExecutableVersions.before.json DataConnectExecutableVersions.after.json ; then + echo "${json_file} is up-to-date" + else + echo "${json_file} is NOT up-to-date" + echo "To update it, run: ./gradlew :firebase-dataconnect:updateJson" + if [[ '${{ github.event_name }}' == 'schedule' ]] ; then + echo "FAILING STEP BECAUSE github.event_name==schedule and JSON file is not up-to-date" >&2 + exit 1 + fi + fi + # The "send-notifications" job adds a comment to GitHub Issue # https://github.com/firebase/firebase-android-sdk/issues/6857 with the results of the scheduled # nightly runs. Interested parties can then subscribe to that issue to be aprised of the outcome @@ -364,6 +411,7 @@ jobs: - 'python-ci-lint' - 'python-ci-format' - 'python-ci-type-check' + - 'emulator-versions-outdated-check' if: always() permissions: issues: write @@ -389,6 +437,7 @@ jobs: python-ci-lint:${{ needs.python-ci-lint.result }} python-ci-format:${{ needs.python-ci-format.result }} python-ci-type-check:${{ needs.python-ci-type-check.result }} + emulator-versions-outdated-check:${{ needs.emulator-versions-outdated-check.result }} EOF - uses: ./.github/actions/dataconnect-send-notifications diff --git a/firebase-dataconnect/connectors/connectors.gradle.kts b/firebase-dataconnect/connectors/connectors.gradle.kts index 3244c8a39f3..b4e912a8aed 100644 --- a/firebase-dataconnect/connectors/connectors.gradle.kts +++ b/firebase-dataconnect/connectors/connectors.gradle.kts @@ -14,7 +14,6 @@ * limitations under the License. */ -import com.google.firebase.dataconnect.gradle.plugin.UpdateDataConnectExecutableVersionsTask import org.gradle.kotlin.dsl.withType import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.tasks.KotlinJvmCompile @@ -113,98 +112,3 @@ tasks.withType().configureEach { compilerOptions.freeCompilerArgs.add("-Xexplicit-api=strict") } } - -// Adds a Gradle task that updates the JSON file that stores the list of Data Connect -// executable versions. -// -// Example 1: Add versions 1.4.3 and 1.4.4 to the JSON file, and set 1.4.4 as the default: -// ../../gradlew -Pversions=1.4.3,1.4.4 -PdefaultVersion=1.4.4 updateJson --info -// -// Example 2: Add version 1.2.3 to the JSON file, but do not change the default version: -// ../../gradlew -Pversion=1.2.3 updateJson --info -// -// The `--info` argument can be omitted; it merely controls the level of log output. -tasks.register("updateJson") { - outputs.upToDateWhen { false } - jsonFile.set( - project.layout.projectDirectory.file( - "../gradleplugin/plugin/src/main/resources/com/google/firebase/dataconnect/gradle/" + - "plugin/DataConnectExecutableVersions.json" - ) - ) - workDirectory.set(project.layout.buildDirectory.dir("updateJson")) - - val propertyNames = - object { - val version = "version" - val versions = "versions" - val updateMode = "updateMode" - val defaultVersion = "defaultVersion" - } - - val singleVersion: String? = project.providers.gradleProperty(propertyNames.version).orNull - val multipleVersions: List? = - project.providers.gradleProperty(propertyNames.versions).orNull?.split(',') - versions.set( - buildList { - singleVersion?.let { add(it) } - multipleVersions?.let { addAll(it) } - } - ) - - doFirst { - if (versions.get().isEmpty()) { - logger.warn( - "WARNING: no '${propertyNames.version}' or '${propertyNames.versions}' specified " + - "for task '$name'; no versions will be added to ${jsonFile.get()}. " + - "Try specifying something like '-P${propertyNames.version}=1.2.3' or " + - "'-P${propertyNames.versions}=1.2.3,4.5.6' on the gradle command line " + - "if you want to add versions (warning code bm6d5ezxzd)" - ) - } - } - - updateMode.set( - project.providers.gradleProperty(propertyNames.updateMode).map { - when (it) { - "overwrite" -> UpdateDataConnectExecutableVersionsTask.UpdateMode.Overwrite - "update" -> UpdateDataConnectExecutableVersionsTask.UpdateMode.Update - else -> - throw Exception( - "Invalid '${propertyNames.updateMode}' specified for task '$name': $it. " + - "Valid values are 'update' and 'overwrite'. " + - "Try specifying '-P${propertyNames.updateMode}=update' or " + - "'-P${propertyNames.updateMode}=overwrite' on the gradle command line. " + - "(error code v2e3cfqbnf)" - ) - } - } - ) - - doFirst { - if (!updateMode.isPresent) { - logger.warn( - "WARNING: no '${propertyNames.updateMode}' specified for task '$name'; " + - "the default update mode of 'update' will be used when updating ${jsonFile.get()}. " + - "Try specifying '-P${propertyNames.updateMode}=update' or " + - "'-P${propertyNames.updateMode}=overwrite' on the gradle command line " + - "if you want a different update mode, or just want to be explicit about " + - "which update mode is in effect (warning code tjyscqmdne)" - ) - } - } - - defaultVersion.set(project.providers.gradleProperty(propertyNames.defaultVersion)) - - doFirst { - if (!defaultVersion.isPresent) { - logger.warn( - "WARNING: no '${propertyNames.defaultVersion}' specified for task '$name'; " + - "the default version will not be updated in ${jsonFile.get()}. " + - "Try specifying something like '-P${propertyNames.defaultVersion}=1.2.3' " + - "on the gradle command line if you want to update the default version " + - "(warning code vqrbrktx9f)" - ) - } - } -} diff --git a/firebase-dataconnect/firebase-dataconnect.gradle.kts b/firebase-dataconnect/firebase-dataconnect.gradle.kts index afc2c2ec6f7..2efaca64808 100644 --- a/firebase-dataconnect/firebase-dataconnect.gradle.kts +++ b/firebase-dataconnect/firebase-dataconnect.gradle.kts @@ -14,6 +14,7 @@ * limitations under the License. */ +import com.google.firebase.dataconnect.gradle.plugin.UpdateDataConnectExecutableVersionsTask import org.gradle.kotlin.dsl.withType import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.tasks.KotlinJvmCompile @@ -24,6 +25,7 @@ plugins { id("com.google.protobuf") id("copy-google-services") alias(libs.plugins.kotlinx.serialization) + id("com.google.firebase.dataconnect.gradle.plugin") apply false } firebaseLibrary { @@ -163,3 +165,16 @@ tasks.withType().configureEach { compilerOptions.freeCompilerArgs.add("-Xexplicit-api=strict") } } + +// Registers a Gradle task that updates the JSON file that stores the list of Data Connect +// executable versions. The task gets the list of all versions from the Internet and then +// updates the JSON file with their sizes and hashes. +tasks.register("updateJson") { + jsonFile.set( + file( + "gradleplugin/plugin/src/main/resources/com/google/firebase/dataconnect/gradle/" + + "plugin/DataConnectExecutableVersions.json" + ) + ) + workDirectory.set(project.layout.buildDirectory.dir("updateJson")) +} diff --git a/firebase-dataconnect/gradleplugin/plugin/build.gradle.kts b/firebase-dataconnect/gradleplugin/plugin/build.gradle.kts index fa34c69e7d6..b63e4ca4733 100644 --- a/firebase-dataconnect/gradleplugin/plugin/build.gradle.kts +++ b/firebase-dataconnect/gradleplugin/plugin/build.gradle.kts @@ -28,6 +28,8 @@ dependencies { implementation(gradleKotlinDsl()) implementation(firebaseLibs.kotlinx.serialization.core) implementation(firebaseLibs.kotlinx.serialization.json) + implementation("com.google.cloud:google-cloud-storage:2.56.0") + implementation("io.github.z4kn4fein:semver:3.0.0") } gradlePlugin { diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableDownloadTask.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableDownloadTask.kt index 10e71c5303c..be6653faff5 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableDownloadTask.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableDownloadTask.kt @@ -110,7 +110,7 @@ abstract class DataConnectExecutableDownloadTask : DefaultTask() { .sorted() .joinToString(", ") val applicableVersions = - allVersions.filter { it.version == version && it.os == operatingSystem } + allVersions.filter { it.version.toString() == version && it.os == operatingSystem } if (applicableVersions.isEmpty()) { val message = diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersionRegistry.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersionRegistry.kt index 7f95885c4cd..a69517c1cbf 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersionRegistry.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersionRegistry.kt @@ -15,6 +15,8 @@ */ package com.google.firebase.dataconnect.gradle.plugin +import io.github.z4kn4fein.semver.LooseVersionSerializer +import io.github.z4kn4fein.semver.Version import java.io.InputStream import java.io.OutputStream import kotlinx.serialization.ExperimentalSerializationApi @@ -38,6 +40,7 @@ object DataConnectExecutableVersionsRegistry { Json { prettyPrint = true prettyPrintIndent = " " + allowTrailingComma = true } } @@ -59,13 +62,13 @@ object DataConnectExecutableVersionsRegistry { @Serializable data class Root( - val defaultVersion: String, + @Serializable(with = LooseVersionSerializer::class) val defaultVersion: Version, val versions: List, ) @Serializable data class VersionInfo( - val version: String, + @Serializable(with = LooseVersionSerializer::class) val version: Version, @Serializable(with = OperatingSystemSerializer::class) val os: OperatingSystem, val size: Long, val sha512DigestHex: String, @@ -79,24 +82,24 @@ object DataConnectExecutableVersionsRegistry { ) override fun deserialize(decoder: Decoder): OperatingSystem = - when (val name = decoder.decodeString()) { - "windows" -> OperatingSystem.Windows - "macos" -> OperatingSystem.MacOS - "linux" -> OperatingSystem.Linux - else -> - throw DataConnectGradleException( + decoder.decodeString().let { serializedValue -> + OperatingSystem.entries.singleOrNull { it.serializedValue == serializedValue } + ?: throw DataConnectGradleException( "nd5z2jk4hr", - "Unknown operating system: $name (must be windows, macos, or linux)" + "Unknown operating system: $serializedValue " + + "(must be one of ${OperatingSystem.entries.joinToString { it.serializedValue }})" ) } override fun serialize(encoder: Encoder, value: OperatingSystem) = - encoder.encodeString( - when (value) { - OperatingSystem.Windows -> "windows" - OperatingSystem.MacOS -> "macos" - OperatingSystem.Linux -> "linux" - } - ) + encoder.encodeString(value.serializedValue) } + + val OperatingSystem.serializedValue: String + get() = + when (this) { + OperatingSystem.Windows -> "windows" + OperatingSystem.MacOS -> "macos" + OperatingSystem.Linux -> "linux" + } } diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt index da7884274dd..b9bae033bfa 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt @@ -49,7 +49,7 @@ class DataConnectProviders( val defaultVersion: Provider = project.provider { val root = DataConnectExecutableVersionsRegistry.load() - DataConnectExecutable.Version(root.defaultVersion) + DataConnectExecutable.Version(root.defaultVersion.toString()) } valueFromLocalSettings diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt index 099324fbc04..3aa2d519de4 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt @@ -16,163 +16,265 @@ package com.google.firebase.dataconnect.gradle.plugin -import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableDownloadTask.Companion.downloadDataConnectExecutable -import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableDownloadTask.FileInfo +import com.google.cloud.storage.Blob +import com.google.cloud.storage.Storage.BlobListOption +import com.google.cloud.storage.StorageException +import com.google.cloud.storage.StorageOptions +import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableVersionsRegistry.serializedValue +import io.github.z4kn4fein.semver.Version +import io.github.z4kn4fein.semver.toVersion +import io.github.z4kn4fein.semver.toVersionOrNull import java.io.File +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import java.time.format.FormatStyle import javax.inject.Inject import kotlin.random.Random import org.gradle.api.DefaultTask import org.gradle.api.file.DirectoryProperty -import org.gradle.api.file.RegularFileProperty -import org.gradle.api.provider.ListProperty import org.gradle.api.provider.Property import org.gradle.api.tasks.Input -import org.gradle.api.tasks.InputFile import org.gradle.api.tasks.Internal -import org.gradle.api.tasks.Optional import org.gradle.api.tasks.TaskAction import org.gradle.process.ExecOperations @Suppress("unused") abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { - @get:InputFile abstract val jsonFile: RegularFileProperty - - @get:Input abstract val versions: ListProperty - - @get:Input @get:Optional abstract val defaultVersion: Property - - @get:Input @get:Optional abstract val updateMode: Property + @get:Input abstract val jsonFile: Property @get:Internal abstract val workDirectory: DirectoryProperty @get:Inject abstract val execOperations: ExecOperations + init { + // Make sure the task ALWAYS runs and is never skipped because Gradle deems it "up to date". + outputs.upToDateWhen { false } + } + @TaskAction fun run() { - val jsonFile: File = jsonFile.get().asFile - val versions: List = versions.get() - val defaultVersion: String? = defaultVersion.orNull - val updateMode: UpdateMode? = updateMode.orNull + val jsonFile: File = jsonFile.get() val workDirectory: File = workDirectory.get().asFile logger.info("jsonFile={}", jsonFile.absolutePath) - logger.info("versions={}", versions) - logger.info("defaultVersion={}", defaultVersion) - logger.info("updateMode={}", updateMode) - logger.info("workDirectory={}", workDirectory) - - var json: DataConnectExecutableVersionsRegistry.Root = - if (updateMode == UpdateMode.Overwrite) { - DataConnectExecutableVersionsRegistry.Root( - defaultVersion = "", - versions = emptyList() - ) - } else { - logger.info("Loading JSON file {}", jsonFile.absolutePath) - DataConnectExecutableVersionsRegistry.load(jsonFile) + logger.info("workDirectory={}", workDirectory.absolutePath) + + logger.lifecycle("Loading executable versions from registry file: {}", jsonFile.absolutePath) + val registry = DataConnectExecutableVersionsRegistry.load(jsonFile) + logger.info( + "Loaded {} executable versions from registry file {} (default={}): {}", + registry.versions.size, + jsonFile.absolutePath, + registry.defaultVersion, + registry.versions.sortedWith(versionInfoComparator).joinToString { + "${it.version}-${it.os.serializedValue}" } + ) - if (defaultVersion !== null) { - json = json.copy(defaultVersion = defaultVersion) - } + val cloudStorageVersions = downloadVersionInfoFromCloudStorage() - for (version in versions) { - val windowsExecutable = download(version, OperatingSystem.Windows, workDirectory) - val macosExecutable = download(version, OperatingSystem.MacOS, workDirectory) - val linuxExecutable = download(version, OperatingSystem.Linux, workDirectory) - json = json.withVersions(version, windowsExecutable, macosExecutable, linuxExecutable) + val unknownVersions = + cloudStorageVersions + .filterNotIn(registry) + .sortedWith( + compareBy { it.version } + .thenByDescending { it.operatingSystem.serializedValue } + ) + if (unknownVersions.isEmpty()) { + logger.lifecycle( + "Not updating {} since it already contains all versions.", + jsonFile.absolutePath + ) + return } - logger.info( - "Writing information about versions {} to file with updateMode={}: {}", - versions.joinToString(", "), - updateMode, - jsonFile.absolutePath + logger.lifecycle( + "Downloading details for {} versions missing from registry file: {}", + unknownVersions.size, + unknownVersions.joinToString { "${it.version}-${it.operatingSystem.serializedValue}" } ) - DataConnectExecutableVersionsRegistry.save(json, jsonFile) - } + val unknownVersionInfos = unknownVersions.map { it.toRegistryVersionInfo(workDirectory) } + val updatedRegistry = registry.updatedWith(unknownVersionInfos) - private fun DataConnectExecutableVersionsRegistry.Root.withVersions( - version: String, - windows: DownloadedFile, - macos: DownloadedFile, - linux: DownloadedFile - ): DataConnectExecutableVersionsRegistry.Root { - data class UpdatedVersion( - val operatingSystem: OperatingSystem, - val sizeInBytes: Long, - val sha512DigestHex: String, - ) { - constructor( - operatingSystem: OperatingSystem, - downloadedFile: DownloadedFile - ) : this(operatingSystem, downloadedFile.sizeInBytes, downloadedFile.sha512DigestHex) - } - val updatedVersions = - listOf( - UpdatedVersion(OperatingSystem.Windows, windows), - UpdatedVersion(OperatingSystem.MacOS, macos), - UpdatedVersion(OperatingSystem.Linux, linux), + logger.lifecycle( + "Updating {} with {} versions: {}", + jsonFile.absolutePath, + unknownVersions.size, + unknownVersions.joinToString { "${it.version}-${it.operatingSystem.serializedValue}" } + ) + + if (updatedRegistry.defaultVersion == registry.defaultVersion) { + logger.lifecycle( + "Not updating default version in {} because it is already the latest version: {}", + jsonFile.absolutePath, + updatedRegistry.defaultVersion ) + } else { + logger.lifecycle( + "Updating default version in {} to the latest version: {} (was {})", + jsonFile.absolutePath, + updatedRegistry.defaultVersion, + registry.defaultVersion + ) + } - val newVersions = versions.toMutableList() - for (updatedVersion in updatedVersions) { - val index = - newVersions.indexOfFirst { - it.version == version && it.os == updatedVersion.operatingSystem + DataConnectExecutableVersionsRegistry.save(updatedRegistry, jsonFile) + } + + private fun downloadVersionInfoFromCloudStorage(): Set { + val storage = StorageOptions.getDefaultInstance().service + val bucketName = "firemat-preview-drop" + logger.lifecycle("Finding all Data Connect executable versions in GCS bucket: {}", bucketName) + val bucket = + storage + .runCatching { get(bucketName) } + .onFailure { e -> + if ( + e is StorageException && + e.cause.let { + it is com.google.api.client.http.HttpResponseException && + (it.statusCode == 401 || it.statusCode == 403) + } + ) { + logger.error( + "ERROR: 401/403 error returned from Google Cloud Storage; " + + "try running \"gcloud auth application-default login\" and/or unsetting the " + + "GOOGLE_APPLICATION_CREDENTIALS environment variable to fix" + ) + } } - if (index >= 0) { - val newVersion = - newVersions[index].copy( - size = updatedVersion.sizeInBytes, - sha512DigestHex = updatedVersion.sha512DigestHex, - ) - newVersions[index] = newVersion - } else { - val newVersion = - DataConnectExecutableVersionsRegistry.VersionInfo( - version = version, - os = updatedVersion.operatingSystem, - size = updatedVersion.sizeInBytes, - sha512DigestHex = updatedVersion.sha512DigestHex, + .getOrThrow() + ?: throw DataConnectGradleException("bvkxzp2esg", "GCS bucket not found: $bucketName") + + val invalidVersions = setOf("1.15.0".toVersion()) + val minVersion = "1.3.4".toVersion() + + val blobs = bucket.list(BlobListOption.prefix("emulator/")) + val regex = ".*dataconnect-emulator-([^-]+)-v(.*)".toRegex() + val dataConnectExecutableBinaries = + blobs + .iterateAll() + .mapNotNull { + logger.debug("[av7zhespw2] Found Data Connect executable file: {}", it.name) + val match = + regex.matchEntire(it.name) + ?: run { + logger.debug( + "[p4vjjcp2kq] Ignoring Data Connect executable file: {} " + + "(does not match regex: {})", + it.name, + regex + ) + return@mapNotNull null + } + CloudStorageVersionInfo( + version = + run { + val versionString = match.groups[2]?.value + versionString?.toVersionOrNull(strict = false) + ?: run { + logger.info( + "WARNING: Ignoring Data Connect executable file: {} " + + "(invalid version: {} (in match for regex {}))", + it.name, + versionString, + regex + ) + return@mapNotNull null + } + }, + operatingSystem = + when (val operatingSystemString = match.groups[1]?.value) { + "linux" -> OperatingSystem.Linux + "macos" -> OperatingSystem.MacOS + "windows" -> OperatingSystem.Windows + else -> { + logger.info( + "WARNING: Ignoring Data Connect executable file: {} " + + "(unknown operating system name: {} (in match for regex {}))", + it.name, + operatingSystemString, + regex + ) + return@mapNotNull null + } + }, + blob = it, ) - newVersions.add(newVersion) - } - } + } + .filter { it.version >= minVersion } + .filterNot { invalidVersions.contains(it.version) } + .toSet() - return this.copy(versions = newVersions.toList()) + return dataConnectExecutableBinaries } - private fun download( - version: String, - operatingSystem: OperatingSystem, - outputDirectory: File - ): DownloadedFile { - val randomId = Random.nextAlphanumericString(length = 20) - val outputFile = - File(outputDirectory, "DataConnectToolkit_${version}_${operatingSystem}_$randomId") + private data class CloudStorageVersionInfo( + val version: Version, + val operatingSystem: OperatingSystem, + val blob: Blob, + ) - downloadDataConnectExecutable(version, operatingSystem, outputFile, execOperations) + private fun CloudStorageVersionInfo.toRegistryVersionInfo( + workDirectory: File + ): DataConnectExecutableVersionsRegistry.VersionInfo { + val dateFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.LONG) - logger.info("Calculating SHA512 hash of file: {}", outputFile.absolutePath) - val fileInfo = FileInfo.forFile(outputFile) + logger.lifecycle( + "Downloading version {} ({} bytes, created {})", + "$version-${operatingSystem.serializedValue}", + blob.size.toStringWithThousandsSeparator(), + dateFormatter.format(blob.createTimeOffsetDateTime.atZoneSameInstant(ZoneId.systemDefault())) + ) + workDirectory.mkdirs() + val outputFile = File(workDirectory, Random.nextAlphanumericString(12)) + outputFile.outputStream().use { dest -> blob.downloadTo(dest) } - return DownloadedFile( - file = outputFile, - sizeInBytes = fileInfo.sizeInBytes, - sha512DigestHex = fileInfo.sha512DigestHex, + val fileInfo = DataConnectExecutableDownloadTask.FileInfo.forFile(outputFile) + outputFile.delete() + check(fileInfo.sizeInBytes == blob.size) { + "fileInfo.sizeInBytes!=blob.size (${fileInfo.sizeInBytes}!=${blob.size}) and this should " + + "never happen; if it _does_ happen it _could_ indicate a compromised " + + "downloaded binary [y5967yd2cf]" + } + return DataConnectExecutableVersionsRegistry.VersionInfo( + version, + operatingSystem, + fileInfo.sizeInBytes, + fileInfo.sha512DigestHex ) } - private data class DownloadedFile( - val file: File, - val sizeInBytes: Long, - val sha512DigestHex: String, - ) + private companion object { + + val versionInfoComparator = + compareBy { it.version } + .thenByDescending { it.os.serializedValue } - enum class UpdateMode { - Overwrite, - Update + /** + * Creates a returns a new list that contains all elements of the receiving [Iterable] that are + * not in the given registry. + */ + private fun Iterable.filterNotIn( + registry: DataConnectExecutableVersionsRegistry.Root + ): List = filterNot { cloudStorageVersion -> + registry.versions.any { + it.version == cloudStorageVersion.version && it.os == cloudStorageVersion.operatingSystem + } + } + + private fun DataConnectExecutableVersionsRegistry.Root.updatedWith( + updatedVersions: Iterable + ): DataConnectExecutableVersionsRegistry.Root { + val mergedVersions = buildList { + addAll(versions) + for (version in updatedVersions) { + val index = indexOfLast { versionInfoComparator.compare(it, version) < 0 } + add(index + 1, version) + } + } + return copy(defaultVersion = mergedVersions.maxOf { it.version }, versions = mergedVersions) + } } } diff --git a/firebase-dataconnect/scripts/missingversions.py b/firebase-dataconnect/scripts/missingversions.py deleted file mode 100755 index 80ee9ecfe58..00000000000 --- a/firebase-dataconnect/scripts/missingversions.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run this script in the root directory of this Git repository to -# determine the versions of the Data Connect Toolkit that are missing -# from the DataConnectExecutableVersions.json file. The final output -# of this script will be the gradle command to run to update the json -# file. -# -# Make sure to run "pip install packaging" before running this script. - -import json -import os -from packaging.version import Version -import re -import subprocess -import tempfile - -regex = re.compile(r".*dataconnect-emulator-linux-v(\d+\.\d+\.\d+)") -json_path = os.path.abspath("firebase-dataconnect/gradleplugin/plugin/src/main/resources/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersions.json") -min_version = Version("1.3.4") -bucket = "gs://firemat-preview-drop/emulator/" - -args = ["gsutil", "ls", "-r", bucket] -print("Getting versions by running: " + subprocess.list2cmdline(args)) -with tempfile.TemporaryFile() as f: - subprocess.check_call(args, stdout=f) - f.seek(0) - filenames = f.read().decode("utf8", errors="strict").splitlines() - -filename_matches = [regex.fullmatch(filename) for filename in filenames] -versions_set = set(match.group(1) for match in filename_matches if match is not None) -all_versions = sorted(versions_set, key=Version) -versions = [version for version in all_versions if Version(version) >= min_version] - -try: - invalid_version_index = versions.index("1.15.0") -except ValueError: - pass -else: - versions.pop(invalid_version_index) - -print(f"Found {len(versions)} versions greater than {min_version}: {versions!r}") -print() - -with open(json_path, "rb") as f: - known_versions_map = json.load(f) -known_versions_set = frozenset(version_info["version"] for version_info in known_versions_map["versions"]) -known_versions = sorted(known_versions_set, key=Version) -print(f"Found {len(known_versions)} versions in {os.path.basename(json_path)}: {known_versions!r}") -print() - -missing_versions = [version for version in versions if version not in known_versions] -print(f"Found {len(missing_versions)} missing versions in {os.path.basename(json_path)}: {missing_versions!r}") -print() - -print(f"Run this gradle command to update {json_path}:") -print(f"./gradlew :firebase-dataconnect:connectors:updateJson -Pversions={",".join(missing_versions)} -PdefaultVersion={versions[-1]}") diff --git a/firebase-dataconnect/scripts/update_versions_json.sh b/firebase-dataconnect/scripts/update_versions_json.sh new file mode 100755 index 00000000000..ff04bbc20f3 --- /dev/null +++ b/firebase-dataconnect/scripts/update_versions_json.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +PROJECT_ROOT_DIR="$(dirname "$0")/../.." +readonly PROJECT_ROOT_DIR + +readonly args=( + "${PROJECT_ROOT_DIR}/gradlew" + "-p" + "${PROJECT_ROOT_DIR}" + "--configure-on-demand" + "$@" + ":firebase-dataconnect:updateJson" +) + +echo "${args[*]}" +exec "${args[@]}" From aac4ddd6e49fdd2492d3fa6e75943a16ac327abe Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Mon, 8 Sep 2025 23:42:01 -0400 Subject: [PATCH 2/9] .github/workflows/dataconnect.yml: forgot to check out the code [:facepalm:] --- .github/workflows/dataconnect.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index b2d98b214de..ac95655da5f 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -355,14 +355,21 @@ jobs: continue-on-error: false runs-on: ubuntu-latest steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + show-progress: false + - uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0 with: java-version: ${{ env.FDC_JAVA_VERSION }} distribution: temurin + - uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8 with: credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }} + - uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 + - name: Gradle updateJson run: | set -euo pipefail From 93498e85cf8db71dfc873c5cdbaa9cc9a3413d3a Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Mon, 8 Sep 2025 23:45:24 -0400 Subject: [PATCH 3/9] use -x instead of -v --- .github/workflows/dataconnect.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index ac95655da5f..989864aeabf 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -373,7 +373,7 @@ jobs: - name: Gradle updateJson run: | set -euo pipefail - set -v + set -x # Speed up build times and also avoid configuring firebase-crashlytics-ndk # which is finicky integrating with the Android NDK. From 980b11ce386a9bcdbce7bf606f5e1712272ad3e2 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Mon, 8 Sep 2025 23:55:10 -0400 Subject: [PATCH 4/9] yml cleanup --- .github/workflows/dataconnect.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index 989864aeabf..bdae0e01f7f 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -364,6 +364,17 @@ jobs: java-version: ${{ env.FDC_JAVA_VERSION }} distribution: temurin + - name: Restore Gradle Cache + uses: actions/cache/restore@d4323d4df104b026a6aa633fdb11d772146be0bf # 4.2.2 + if: github.event_name != 'schedule' + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-cache-jqnvfzw6w7-${{ github.run_id }} + restore-keys: | + gradle-cache-jqnvfzw6w7- + - uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8 with: credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }} @@ -371,6 +382,7 @@ jobs: - uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 - name: Gradle updateJson + id: update-json run: | set -euo pipefail set -x @@ -392,15 +404,19 @@ jobs: if diff DataConnectExecutableVersions.before.json DataConnectExecutableVersions.after.json ; then echo "${json_file} is up-to-date" + echo "diff_result=same" >> "$GITHUB_OUTPUT" else echo "${json_file} is NOT up-to-date" echo "To update it, run: ./gradlew :firebase-dataconnect:updateJson" - if [[ '${{ github.event_name }}' == 'schedule' ]] ; then - echo "FAILING STEP BECAUSE github.event_name==schedule and JSON file is not up-to-date" >&2 - exit 1 - fi + echo "diff_result=different" >> "$GITHUB_OUTPUT" fi + - name: "Fail if DataConnectExecutableVersions.json is not up-to-date during scheduled runs" + if: github.event_name == 'schedule' && steps.update-json.outputs.diff_result != 'same' + run: | + echo "Force-failing because DataConnectExecutableVersions.json is NOT up-to-date" >&2 + exit 1 + # The "send-notifications" job adds a comment to GitHub Issue # https://github.com/firebase/firebase-android-sdk/issues/6857 with the results of the scheduled # nightly runs. Interested parties can then subscribe to that issue to be aprised of the outcome From 4dd4ea7c51e7148392e4ca7a49e10f39df87ecad Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Mon, 8 Sep 2025 23:55:52 -0400 Subject: [PATCH 5/9] .github/workflows/dataconnect.yml: remove "emulator-versions-outdated-check" (add it back in a follow-up PR) --- .github/workflows/dataconnect.yml | 72 ------------------------------- 1 file changed, 72 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index bdae0e01f7f..aef25709b06 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -347,76 +347,6 @@ jobs: working-directory: firebase-dataconnect/ci run: pyright --warnings --stats - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - show-progress: false - - emulator-versions-outdated-check: - continue-on-error: false - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - show-progress: false - - - uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0 - with: - java-version: ${{ env.FDC_JAVA_VERSION }} - distribution: temurin - - - name: Restore Gradle Cache - uses: actions/cache/restore@d4323d4df104b026a6aa633fdb11d772146be0bf # 4.2.2 - if: github.event_name != 'schedule' - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: gradle-cache-jqnvfzw6w7-${{ github.run_id }} - restore-keys: | - gradle-cache-jqnvfzw6w7- - - - uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8 - with: - credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }} - - - uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 - - - name: Gradle updateJson - id: update-json - run: | - set -euo pipefail - set -x - - # Speed up build times and also avoid configuring firebase-crashlytics-ndk - # which is finicky integrating with the Android NDK. - echo >> gradle.properties - echo "org.gradle.configureondemand=true" >> gradle.properties - - readonly json_file='firebase-dataconnect/gradleplugin/plugin/src/main/resources/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersions.json' - cp "${json_file}" '${{ runner.temp }}/DataConnectExecutableVersions.before.json' - - ./gradlew \ - --warning-mode all \ - ${{ (inputs.gradleInfoLog && '--info') || '' }} \ - :firebase-dataconnect:updateJson - - cp "${json_file}" '${{ runner.temp }}/DataConnectExecutableVersions.after.json' - - if diff DataConnectExecutableVersions.before.json DataConnectExecutableVersions.after.json ; then - echo "${json_file} is up-to-date" - echo "diff_result=same" >> "$GITHUB_OUTPUT" - else - echo "${json_file} is NOT up-to-date" - echo "To update it, run: ./gradlew :firebase-dataconnect:updateJson" - echo "diff_result=different" >> "$GITHUB_OUTPUT" - fi - - - name: "Fail if DataConnectExecutableVersions.json is not up-to-date during scheduled runs" - if: github.event_name == 'schedule' && steps.update-json.outputs.diff_result != 'same' - run: | - echo "Force-failing because DataConnectExecutableVersions.json is NOT up-to-date" >&2 - exit 1 - # The "send-notifications" job adds a comment to GitHub Issue # https://github.com/firebase/firebase-android-sdk/issues/6857 with the results of the scheduled # nightly runs. Interested parties can then subscribe to that issue to be aprised of the outcome @@ -434,7 +364,6 @@ jobs: - 'python-ci-lint' - 'python-ci-format' - 'python-ci-type-check' - - 'emulator-versions-outdated-check' if: always() permissions: issues: write @@ -460,7 +389,6 @@ jobs: python-ci-lint:${{ needs.python-ci-lint.result }} python-ci-format:${{ needs.python-ci-format.result }} python-ci-type-check:${{ needs.python-ci-type-check.result }} - emulator-versions-outdated-check:${{ needs.emulator-versions-outdated-check.result }} EOF - uses: ./.github/actions/dataconnect-send-notifications From abdf9684f046fdeacdae9b21e58828895d27f43c Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 9 Sep 2025 13:34:20 -0400 Subject: [PATCH 6/9] firebase-dataconnect.gradle.kts: use `DataConnectExecutableVersionsRegistry.PATH` in path to the json file --- firebase-dataconnect/firebase-dataconnect.gradle.kts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/firebase-dataconnect/firebase-dataconnect.gradle.kts b/firebase-dataconnect/firebase-dataconnect.gradle.kts index 2efaca64808..d4da49a670c 100644 --- a/firebase-dataconnect/firebase-dataconnect.gradle.kts +++ b/firebase-dataconnect/firebase-dataconnect.gradle.kts @@ -14,6 +14,7 @@ * limitations under the License. */ +import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableVersionsRegistry import com.google.firebase.dataconnect.gradle.plugin.UpdateDataConnectExecutableVersionsTask import org.gradle.kotlin.dsl.withType import org.jetbrains.kotlin.gradle.dsl.JvmTarget @@ -171,10 +172,7 @@ tasks.withType().configureEach { // updates the JSON file with their sizes and hashes. tasks.register("updateJson") { jsonFile.set( - file( - "gradleplugin/plugin/src/main/resources/com/google/firebase/dataconnect/gradle/" + - "plugin/DataConnectExecutableVersions.json" - ) + file("gradleplugin/plugin/src/main/resources/${DataConnectExecutableVersionsRegistry.PATH}") ) workDirectory.set(project.layout.buildDirectory.dir("updateJson")) } From f3d343982c5464821ab700f03777299a1abcf10e Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 9 Sep 2025 15:33:40 -0400 Subject: [PATCH 7/9] UpdateDataConnectExecutableVersionsTask.kt: refactor for improved readability --- ...UpdateDataConnectExecutableVersionsTask.kt | 257 ++++++++++-------- 1 file changed, 137 insertions(+), 120 deletions(-) diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt index 3aa2d519de4..1accefcce05 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt @@ -17,9 +17,12 @@ package com.google.firebase.dataconnect.gradle.plugin import com.google.cloud.storage.Blob +import com.google.cloud.storage.Bucket +import com.google.cloud.storage.Storage import com.google.cloud.storage.Storage.BlobListOption import com.google.cloud.storage.StorageException import com.google.cloud.storage.StorageOptions +import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableVersionsRegistry.VersionInfo import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableVersionsRegistry.serializedValue import io.github.z4kn4fein.semver.Version import io.github.z4kn4fein.semver.toVersion @@ -67,21 +70,21 @@ abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { registry.versions.size, jsonFile.absolutePath, registry.defaultVersion, - registry.versions.sortedWith(versionInfoComparator).joinToString { - "${it.version}-${it.os.serializedValue}" - } + registry.versions.sortedWith(versionInfoComparator).toLogString() ) - val cloudStorageVersions = downloadVersionInfoFromCloudStorage() + val cloudStorageVersions: Set = + StorageOptions.getDefaultInstance() + .service + .getDataConnectExecutablesBucket() + .list(BlobListOption.prefix("emulator/")) + .iterateAll() + .mapNotNull { it.toCloudStorageVersionInfoOrNull() } + .toSet() - val unknownVersions = - cloudStorageVersions - .filterNotIn(registry) - .sortedWith( - compareBy { it.version } - .thenByDescending { it.operatingSystem.serializedValue } - ) - if (unknownVersions.isEmpty()) { + val cloudStorageVersionsMissingFromRegistry: List = + cloudStorageVersions.filterNotIn(registry).sortedWith(cloudStorageVersionInfoComparator) + if (cloudStorageVersionsMissingFromRegistry.isEmpty()) { logger.lifecycle( "Not updating {} since it already contains all versions.", jsonFile.absolutePath @@ -91,17 +94,20 @@ abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { logger.lifecycle( "Downloading details for {} versions missing from registry file: {}", - unknownVersions.size, - unknownVersions.joinToString { "${it.version}-${it.operatingSystem.serializedValue}" } + cloudStorageVersionsMissingFromRegistry.size, + cloudStorageVersionsMissingFromRegistry.toLogString() ) - val unknownVersionInfos = unknownVersions.map { it.toRegistryVersionInfo(workDirectory) } - val updatedRegistry = registry.updatedWith(unknownVersionInfos) + + val updatedRegistry = + registry.updatedWith( + cloudStorageVersionsMissingFromRegistry.map { it.toRegistryVersionInfo(workDirectory) } + ) logger.lifecycle( "Updating {} with {} versions: {}", jsonFile.absolutePath, - unknownVersions.size, - unknownVersions.joinToString { "${it.version}-${it.operatingSystem.serializedValue}" } + cloudStorageVersionsMissingFromRegistry.size, + cloudStorageVersionsMissingFromRegistry.toLogString() ) if (updatedRegistry.defaultVersion == registry.defaultVersion) { @@ -122,103 +128,104 @@ abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { DataConnectExecutableVersionsRegistry.save(updatedRegistry, jsonFile) } - private fun downloadVersionInfoFromCloudStorage(): Set { - val storage = StorageOptions.getDefaultInstance().service + private data class CloudStorageVersionInfo( + val version: Version, + val operatingSystem: OperatingSystem, + val blob: Blob, + ) + + private fun Storage.getDataConnectExecutablesBucket(): Bucket { val bucketName = "firemat-preview-drop" logger.lifecycle("Finding all Data Connect executable versions in GCS bucket: {}", bucketName) - val bucket = - storage - .runCatching { get(bucketName) } - .onFailure { e -> - if ( - e is StorageException && - e.cause.let { - it is com.google.api.client.http.HttpResponseException && - (it.statusCode == 401 || it.statusCode == 403) - } - ) { - logger.error( - "ERROR: 401/403 error returned from Google Cloud Storage; " + - "try running \"gcloud auth application-default login\" and/or unsetting the " + - "GOOGLE_APPLICATION_CREDENTIALS environment variable to fix" - ) - } + + return runCatching { get(bucketName) } + .onFailure { e -> + if ( + e is StorageException && + e.cause.let { + it is com.google.api.client.http.HttpResponseException && + (it.statusCode == 401 || it.statusCode == 403) + } + ) { + logger.error( + "ERROR: 401/403 error returned from Google Cloud Storage; " + + "try running \"gcloud auth application-default login\" and/or unsetting the " + + "GOOGLE_APPLICATION_CREDENTIALS environment variable to fix" + ) } - .getOrThrow() - ?: throw DataConnectGradleException("bvkxzp2esg", "GCS bucket not found: $bucketName") + } + .getOrThrow() + ?: throw DataConnectGradleException("bvkxzp2esg", "GCS bucket not found: $bucketName") + } - val invalidVersions = setOf("1.15.0".toVersion()) - val minVersion = "1.3.4".toVersion() + private fun Blob.toCloudStorageVersionInfoOrNull(): CloudStorageVersionInfo? { + logger.debug("[av7zhespw2] Found Data Connect executable file: {}", name) + val match = + fileNameRegex.matchEntire(name) + ?: run { + logger.debug( + "[p4vjjcp2kq] Ignoring Data Connect executable file: {} " + + "(does not match regex: {})", + name, + fileNameRegex + ) + return null + } - val blobs = bucket.list(BlobListOption.prefix("emulator/")) - val regex = ".*dataconnect-emulator-([^-]+)-v(.*)".toRegex() - val dataConnectExecutableBinaries = - blobs - .iterateAll() - .mapNotNull { - logger.debug("[av7zhespw2] Found Data Connect executable file: {}", it.name) - val match = - regex.matchEntire(it.name) - ?: run { - logger.debug( - "[p4vjjcp2kq] Ignoring Data Connect executable file: {} " + - "(does not match regex: {})", - it.name, - regex - ) - return@mapNotNull null - } - CloudStorageVersionInfo( - version = - run { - val versionString = match.groups[2]?.value - versionString?.toVersionOrNull(strict = false) - ?: run { - logger.info( - "WARNING: Ignoring Data Connect executable file: {} " + - "(invalid version: {} (in match for regex {}))", - it.name, - versionString, - regex - ) - return@mapNotNull null - } - }, - operatingSystem = - when (val operatingSystemString = match.groups[1]?.value) { - "linux" -> OperatingSystem.Linux - "macos" -> OperatingSystem.MacOS - "windows" -> OperatingSystem.Windows - else -> { - logger.info( - "WARNING: Ignoring Data Connect executable file: {} " + - "(unknown operating system name: {} (in match for regex {}))", - it.name, - operatingSystemString, - regex - ) - return@mapNotNull null - } - }, - blob = it, + val versionString = match.groups[2]?.value + val version = versionString?.toVersionOrNull(strict = false) + if (version === null) { + logger.info( + "Ignoring Data Connect executable file: {} " + + "(invalid version: {} (in match for regex {}))", + name, + versionString, + fileNameRegex + ) + return null + } + + if (version < minVersion) { + logger.info( + "Ignoring Data Connect executable file: {} " + + "(version {} is less than the minimum version: {})", + name, + versionString, + minVersion + ) + return null + } + + if (version in invalidVersions) { + logger.info( + "Ignoring Data Connect executable file: {} " + "(version {} is a known invalid version)", + name, + versionString + ) + return null + } + + val operatingSystem = + when (val operatingSystemString = match.groups[1]?.value) { + "linux" -> OperatingSystem.Linux + "macos" -> OperatingSystem.MacOS + "windows" -> OperatingSystem.Windows + else -> { + logger.info( + "WARNING: Ignoring Data Connect executable file: {} " + + "(unknown operating system name: {} (in match for regex {}))", + name, + operatingSystemString, + fileNameRegex ) + return null } - .filter { it.version >= minVersion } - .filterNot { invalidVersions.contains(it.version) } - .toSet() + } - return dataConnectExecutableBinaries + return CloudStorageVersionInfo(version, operatingSystem, blob = this) } - private data class CloudStorageVersionInfo( - val version: Version, - val operatingSystem: OperatingSystem, - val blob: Blob, - ) - - private fun CloudStorageVersionInfo.toRegistryVersionInfo( - workDirectory: File - ): DataConnectExecutableVersionsRegistry.VersionInfo { + private fun CloudStorageVersionInfo.toRegistryVersionInfo(workDirectory: File): VersionInfo { val dateFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.LONG) logger.lifecycle( @@ -238,19 +245,31 @@ abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { "never happen; if it _does_ happen it _could_ indicate a compromised " + "downloaded binary [y5967yd2cf]" } - return DataConnectExecutableVersionsRegistry.VersionInfo( - version, - operatingSystem, - fileInfo.sizeInBytes, - fileInfo.sha512DigestHex - ) + return VersionInfo(version, operatingSystem, fileInfo.sizeInBytes, fileInfo.sha512DigestHex) } private companion object { val versionInfoComparator = - compareBy { it.version } - .thenByDescending { it.os.serializedValue } + compareBy { it.version }.thenByDescending { it.os.serializedValue } + + val cloudStorageVersionInfoComparator = + compareBy { it.version } + .thenByDescending { it.operatingSystem.serializedValue } + + @JvmName("toLogStringCloudStorageVersionInfo") + fun Iterable.toLogString(): String = joinToString { + "${it.version}-${it.operatingSystem.serializedValue}" + } + + @JvmName("toLogStringVersionInfo") + fun Iterable.toLogString(): String = joinToString { + "${it.version}-${it.os.serializedValue}" + } + + val invalidVersions = setOf("1.15.0".toVersion()) + val minVersion = "1.3.4".toVersion() + val fileNameRegex = ".*dataconnect-emulator-([^-]+)-v(.*)".toRegex() /** * Creates a returns a new list that contains all elements of the receiving [Iterable] that are @@ -265,16 +284,14 @@ abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { } private fun DataConnectExecutableVersionsRegistry.Root.updatedWith( - updatedVersions: Iterable + updatedVersions: Iterable ): DataConnectExecutableVersionsRegistry.Root { - val mergedVersions = buildList { + val allVersions = buildList { addAll(versions) - for (version in updatedVersions) { - val index = indexOfLast { versionInfoComparator.compare(it, version) < 0 } - add(index + 1, version) - } + addAll(updatedVersions) + sortWith(versionInfoComparator) } - return copy(defaultVersion = mergedVersions.maxOf { it.version }, versions = mergedVersions) + return copy(defaultVersion = allVersions.maxOf { it.version }, versions = allVersions) } } } From f766f3fb979bc68adf565d22d44ef71d9d2b37ad Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 9 Sep 2025 15:37:41 -0400 Subject: [PATCH 8/9] more tiny code cleanup --- .../plugin/UpdateDataConnectExecutableVersionsTask.kt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt index 1accefcce05..e9edaac6d6c 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt @@ -97,11 +97,8 @@ abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { cloudStorageVersionsMissingFromRegistry.size, cloudStorageVersionsMissingFromRegistry.toLogString() ) - - val updatedRegistry = - registry.updatedWith( - cloudStorageVersionsMissingFromRegistry.map { it.toRegistryVersionInfo(workDirectory) } - ) + val missingRegistryVersionInfos = + cloudStorageVersionsMissingFromRegistry.map { it.toRegistryVersionInfo(workDirectory) } logger.lifecycle( "Updating {} with {} versions: {}", @@ -110,6 +107,8 @@ abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { cloudStorageVersionsMissingFromRegistry.toLogString() ) + val updatedRegistry = registry.updatedWith(missingRegistryVersionInfos) + if (updatedRegistry.defaultVersion == registry.defaultVersion) { logger.lifecycle( "Not updating default version in {} because it is already the latest version: {}", From e48756be60094dae4f217f320e52a0ae3fe6e85b Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 9 Sep 2025 15:41:47 -0400 Subject: [PATCH 9/9] minor cleanups found by copilot --- .../plugin/UpdateDataConnectExecutableVersionsTask.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt index e9edaac6d6c..f34d75bb9bf 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt @@ -197,7 +197,7 @@ abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { if (version in invalidVersions) { logger.info( - "Ignoring Data Connect executable file: {} " + "(version {} is a known invalid version)", + "Ignoring Data Connect executable file: {} (version {} is a known invalid version)", name, versionString ) @@ -271,8 +271,8 @@ abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { val fileNameRegex = ".*dataconnect-emulator-([^-]+)-v(.*)".toRegex() /** - * Creates a returns a new list that contains all elements of the receiving [Iterable] that are - * not in the given registry. + * Creates and returns a new list that contains all elements of the receiving [Iterable] that + * are not in the given registry. */ private fun Iterable.filterNotIn( registry: DataConnectExecutableVersionsRegistry.Root