diff --git a/firebase-dataconnect/connectors/connectors.gradle.kts b/firebase-dataconnect/connectors/connectors.gradle.kts index 3244c8a39f3..b4e912a8aed 100644 --- a/firebase-dataconnect/connectors/connectors.gradle.kts +++ b/firebase-dataconnect/connectors/connectors.gradle.kts @@ -14,7 +14,6 @@ * limitations under the License. */ -import com.google.firebase.dataconnect.gradle.plugin.UpdateDataConnectExecutableVersionsTask import org.gradle.kotlin.dsl.withType import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.tasks.KotlinJvmCompile @@ -113,98 +112,3 @@ tasks.withType().configureEach { compilerOptions.freeCompilerArgs.add("-Xexplicit-api=strict") } } - -// Adds a Gradle task that updates the JSON file that stores the list of Data Connect -// executable versions. -// -// Example 1: Add versions 1.4.3 and 1.4.4 to the JSON file, and set 1.4.4 as the default: -// ../../gradlew -Pversions=1.4.3,1.4.4 -PdefaultVersion=1.4.4 updateJson --info -// -// Example 2: Add version 1.2.3 to the JSON file, but do not change the default version: -// ../../gradlew -Pversion=1.2.3 updateJson --info -// -// The `--info` argument can be omitted; it merely controls the level of log output. -tasks.register("updateJson") { - outputs.upToDateWhen { false } - jsonFile.set( - project.layout.projectDirectory.file( - "../gradleplugin/plugin/src/main/resources/com/google/firebase/dataconnect/gradle/" + - "plugin/DataConnectExecutableVersions.json" - ) - ) - workDirectory.set(project.layout.buildDirectory.dir("updateJson")) - - val propertyNames = - object { - val version = "version" - val versions = "versions" - val updateMode = "updateMode" - val defaultVersion = "defaultVersion" - } - - val singleVersion: String? = project.providers.gradleProperty(propertyNames.version).orNull - val multipleVersions: List? = - project.providers.gradleProperty(propertyNames.versions).orNull?.split(',') - versions.set( - buildList { - singleVersion?.let { add(it) } - multipleVersions?.let { addAll(it) } - } - ) - - doFirst { - if (versions.get().isEmpty()) { - logger.warn( - "WARNING: no '${propertyNames.version}' or '${propertyNames.versions}' specified " + - "for task '$name'; no versions will be added to ${jsonFile.get()}. " + - "Try specifying something like '-P${propertyNames.version}=1.2.3' or " + - "'-P${propertyNames.versions}=1.2.3,4.5.6' on the gradle command line " + - "if you want to add versions (warning code bm6d5ezxzd)" - ) - } - } - - updateMode.set( - project.providers.gradleProperty(propertyNames.updateMode).map { - when (it) { - "overwrite" -> UpdateDataConnectExecutableVersionsTask.UpdateMode.Overwrite - "update" -> UpdateDataConnectExecutableVersionsTask.UpdateMode.Update - else -> - throw Exception( - "Invalid '${propertyNames.updateMode}' specified for task '$name': $it. " + - "Valid values are 'update' and 'overwrite'. " + - "Try specifying '-P${propertyNames.updateMode}=update' or " + - "'-P${propertyNames.updateMode}=overwrite' on the gradle command line. " + - "(error code v2e3cfqbnf)" - ) - } - } - ) - - doFirst { - if (!updateMode.isPresent) { - logger.warn( - "WARNING: no '${propertyNames.updateMode}' specified for task '$name'; " + - "the default update mode of 'update' will be used when updating ${jsonFile.get()}. " + - "Try specifying '-P${propertyNames.updateMode}=update' or " + - "'-P${propertyNames.updateMode}=overwrite' on the gradle command line " + - "if you want a different update mode, or just want to be explicit about " + - "which update mode is in effect (warning code tjyscqmdne)" - ) - } - } - - defaultVersion.set(project.providers.gradleProperty(propertyNames.defaultVersion)) - - doFirst { - if (!defaultVersion.isPresent) { - logger.warn( - "WARNING: no '${propertyNames.defaultVersion}' specified for task '$name'; " + - "the default version will not be updated in ${jsonFile.get()}. " + - "Try specifying something like '-P${propertyNames.defaultVersion}=1.2.3' " + - "on the gradle command line if you want to update the default version " + - "(warning code vqrbrktx9f)" - ) - } - } -} diff --git a/firebase-dataconnect/firebase-dataconnect.gradle.kts b/firebase-dataconnect/firebase-dataconnect.gradle.kts index afc2c2ec6f7..d4da49a670c 100644 --- a/firebase-dataconnect/firebase-dataconnect.gradle.kts +++ b/firebase-dataconnect/firebase-dataconnect.gradle.kts @@ -14,6 +14,8 @@ * limitations under the License. */ +import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableVersionsRegistry +import com.google.firebase.dataconnect.gradle.plugin.UpdateDataConnectExecutableVersionsTask import org.gradle.kotlin.dsl.withType import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.tasks.KotlinJvmCompile @@ -24,6 +26,7 @@ plugins { id("com.google.protobuf") id("copy-google-services") alias(libs.plugins.kotlinx.serialization) + id("com.google.firebase.dataconnect.gradle.plugin") apply false } firebaseLibrary { @@ -163,3 +166,13 @@ tasks.withType().configureEach { compilerOptions.freeCompilerArgs.add("-Xexplicit-api=strict") } } + +// Registers a Gradle task that updates the JSON file that stores the list of Data Connect +// executable versions. The task gets the list of all versions from the Internet and then +// updates the JSON file with their sizes and hashes. +tasks.register("updateJson") { + jsonFile.set( + file("gradleplugin/plugin/src/main/resources/${DataConnectExecutableVersionsRegistry.PATH}") + ) + workDirectory.set(project.layout.buildDirectory.dir("updateJson")) +} diff --git a/firebase-dataconnect/gradleplugin/plugin/build.gradle.kts b/firebase-dataconnect/gradleplugin/plugin/build.gradle.kts index fa34c69e7d6..b63e4ca4733 100644 --- a/firebase-dataconnect/gradleplugin/plugin/build.gradle.kts +++ b/firebase-dataconnect/gradleplugin/plugin/build.gradle.kts @@ -28,6 +28,8 @@ dependencies { implementation(gradleKotlinDsl()) implementation(firebaseLibs.kotlinx.serialization.core) implementation(firebaseLibs.kotlinx.serialization.json) + implementation("com.google.cloud:google-cloud-storage:2.56.0") + implementation("io.github.z4kn4fein:semver:3.0.0") } gradlePlugin { diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableDownloadTask.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableDownloadTask.kt index 10e71c5303c..be6653faff5 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableDownloadTask.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableDownloadTask.kt @@ -110,7 +110,7 @@ abstract class DataConnectExecutableDownloadTask : DefaultTask() { .sorted() .joinToString(", ") val applicableVersions = - allVersions.filter { it.version == version && it.os == operatingSystem } + allVersions.filter { it.version.toString() == version && it.os == operatingSystem } if (applicableVersions.isEmpty()) { val message = diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersionRegistry.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersionRegistry.kt index 7f95885c4cd..a69517c1cbf 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersionRegistry.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersionRegistry.kt @@ -15,6 +15,8 @@ */ package com.google.firebase.dataconnect.gradle.plugin +import io.github.z4kn4fein.semver.LooseVersionSerializer +import io.github.z4kn4fein.semver.Version import java.io.InputStream import java.io.OutputStream import kotlinx.serialization.ExperimentalSerializationApi @@ -38,6 +40,7 @@ object DataConnectExecutableVersionsRegistry { Json { prettyPrint = true prettyPrintIndent = " " + allowTrailingComma = true } } @@ -59,13 +62,13 @@ object DataConnectExecutableVersionsRegistry { @Serializable data class Root( - val defaultVersion: String, + @Serializable(with = LooseVersionSerializer::class) val defaultVersion: Version, val versions: List, ) @Serializable data class VersionInfo( - val version: String, + @Serializable(with = LooseVersionSerializer::class) val version: Version, @Serializable(with = OperatingSystemSerializer::class) val os: OperatingSystem, val size: Long, val sha512DigestHex: String, @@ -79,24 +82,24 @@ object DataConnectExecutableVersionsRegistry { ) override fun deserialize(decoder: Decoder): OperatingSystem = - when (val name = decoder.decodeString()) { - "windows" -> OperatingSystem.Windows - "macos" -> OperatingSystem.MacOS - "linux" -> OperatingSystem.Linux - else -> - throw DataConnectGradleException( + decoder.decodeString().let { serializedValue -> + OperatingSystem.entries.singleOrNull { it.serializedValue == serializedValue } + ?: throw DataConnectGradleException( "nd5z2jk4hr", - "Unknown operating system: $name (must be windows, macos, or linux)" + "Unknown operating system: $serializedValue " + + "(must be one of ${OperatingSystem.entries.joinToString { it.serializedValue }})" ) } override fun serialize(encoder: Encoder, value: OperatingSystem) = - encoder.encodeString( - when (value) { - OperatingSystem.Windows -> "windows" - OperatingSystem.MacOS -> "macos" - OperatingSystem.Linux -> "linux" - } - ) + encoder.encodeString(value.serializedValue) } + + val OperatingSystem.serializedValue: String + get() = + when (this) { + OperatingSystem.Windows -> "windows" + OperatingSystem.MacOS -> "macos" + OperatingSystem.Linux -> "linux" + } } diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt index da7884274dd..b9bae033bfa 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt @@ -49,7 +49,7 @@ class DataConnectProviders( val defaultVersion: Provider = project.provider { val root = DataConnectExecutableVersionsRegistry.load() - DataConnectExecutable.Version(root.defaultVersion) + DataConnectExecutable.Version(root.defaultVersion.toString()) } valueFromLocalSettings diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt index 099324fbc04..f34d75bb9bf 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/UpdateDataConnectExecutableVersionsTask.kt @@ -16,163 +16,281 @@ package com.google.firebase.dataconnect.gradle.plugin -import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableDownloadTask.Companion.downloadDataConnectExecutable -import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableDownloadTask.FileInfo +import com.google.cloud.storage.Blob +import com.google.cloud.storage.Bucket +import com.google.cloud.storage.Storage +import com.google.cloud.storage.Storage.BlobListOption +import com.google.cloud.storage.StorageException +import com.google.cloud.storage.StorageOptions +import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableVersionsRegistry.VersionInfo +import com.google.firebase.dataconnect.gradle.plugin.DataConnectExecutableVersionsRegistry.serializedValue +import io.github.z4kn4fein.semver.Version +import io.github.z4kn4fein.semver.toVersion +import io.github.z4kn4fein.semver.toVersionOrNull import java.io.File +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import java.time.format.FormatStyle import javax.inject.Inject import kotlin.random.Random import org.gradle.api.DefaultTask import org.gradle.api.file.DirectoryProperty -import org.gradle.api.file.RegularFileProperty -import org.gradle.api.provider.ListProperty import org.gradle.api.provider.Property import org.gradle.api.tasks.Input -import org.gradle.api.tasks.InputFile import org.gradle.api.tasks.Internal -import org.gradle.api.tasks.Optional import org.gradle.api.tasks.TaskAction import org.gradle.process.ExecOperations @Suppress("unused") abstract class UpdateDataConnectExecutableVersionsTask : DefaultTask() { - @get:InputFile abstract val jsonFile: RegularFileProperty - - @get:Input abstract val versions: ListProperty - - @get:Input @get:Optional abstract val defaultVersion: Property - - @get:Input @get:Optional abstract val updateMode: Property + @get:Input abstract val jsonFile: Property @get:Internal abstract val workDirectory: DirectoryProperty @get:Inject abstract val execOperations: ExecOperations + init { + // Make sure the task ALWAYS runs and is never skipped because Gradle deems it "up to date". + outputs.upToDateWhen { false } + } + @TaskAction fun run() { - val jsonFile: File = jsonFile.get().asFile - val versions: List = versions.get() - val defaultVersion: String? = defaultVersion.orNull - val updateMode: UpdateMode? = updateMode.orNull + val jsonFile: File = jsonFile.get() val workDirectory: File = workDirectory.get().asFile logger.info("jsonFile={}", jsonFile.absolutePath) - logger.info("versions={}", versions) - logger.info("defaultVersion={}", defaultVersion) - logger.info("updateMode={}", updateMode) - logger.info("workDirectory={}", workDirectory) - - var json: DataConnectExecutableVersionsRegistry.Root = - if (updateMode == UpdateMode.Overwrite) { - DataConnectExecutableVersionsRegistry.Root( - defaultVersion = "", - versions = emptyList() - ) - } else { - logger.info("Loading JSON file {}", jsonFile.absolutePath) - DataConnectExecutableVersionsRegistry.load(jsonFile) - } + logger.info("workDirectory={}", workDirectory.absolutePath) - if (defaultVersion !== null) { - json = json.copy(defaultVersion = defaultVersion) - } + logger.lifecycle("Loading executable versions from registry file: {}", jsonFile.absolutePath) + val registry = DataConnectExecutableVersionsRegistry.load(jsonFile) + logger.info( + "Loaded {} executable versions from registry file {} (default={}): {}", + registry.versions.size, + jsonFile.absolutePath, + registry.defaultVersion, + registry.versions.sortedWith(versionInfoComparator).toLogString() + ) + + val cloudStorageVersions: Set = + StorageOptions.getDefaultInstance() + .service + .getDataConnectExecutablesBucket() + .list(BlobListOption.prefix("emulator/")) + .iterateAll() + .mapNotNull { it.toCloudStorageVersionInfoOrNull() } + .toSet() - for (version in versions) { - val windowsExecutable = download(version, OperatingSystem.Windows, workDirectory) - val macosExecutable = download(version, OperatingSystem.MacOS, workDirectory) - val linuxExecutable = download(version, OperatingSystem.Linux, workDirectory) - json = json.withVersions(version, windowsExecutable, macosExecutable, linuxExecutable) + val cloudStorageVersionsMissingFromRegistry: List = + cloudStorageVersions.filterNotIn(registry).sortedWith(cloudStorageVersionInfoComparator) + if (cloudStorageVersionsMissingFromRegistry.isEmpty()) { + logger.lifecycle( + "Not updating {} since it already contains all versions.", + jsonFile.absolutePath + ) + return } - logger.info( - "Writing information about versions {} to file with updateMode={}: {}", - versions.joinToString(", "), - updateMode, - jsonFile.absolutePath + logger.lifecycle( + "Downloading details for {} versions missing from registry file: {}", + cloudStorageVersionsMissingFromRegistry.size, + cloudStorageVersionsMissingFromRegistry.toLogString() ) - DataConnectExecutableVersionsRegistry.save(json, jsonFile) - } + val missingRegistryVersionInfos = + cloudStorageVersionsMissingFromRegistry.map { it.toRegistryVersionInfo(workDirectory) } - private fun DataConnectExecutableVersionsRegistry.Root.withVersions( - version: String, - windows: DownloadedFile, - macos: DownloadedFile, - linux: DownloadedFile - ): DataConnectExecutableVersionsRegistry.Root { - data class UpdatedVersion( - val operatingSystem: OperatingSystem, - val sizeInBytes: Long, - val sha512DigestHex: String, - ) { - constructor( - operatingSystem: OperatingSystem, - downloadedFile: DownloadedFile - ) : this(operatingSystem, downloadedFile.sizeInBytes, downloadedFile.sha512DigestHex) - } - val updatedVersions = - listOf( - UpdatedVersion(OperatingSystem.Windows, windows), - UpdatedVersion(OperatingSystem.MacOS, macos), - UpdatedVersion(OperatingSystem.Linux, linux), + logger.lifecycle( + "Updating {} with {} versions: {}", + jsonFile.absolutePath, + cloudStorageVersionsMissingFromRegistry.size, + cloudStorageVersionsMissingFromRegistry.toLogString() + ) + + val updatedRegistry = registry.updatedWith(missingRegistryVersionInfos) + + if (updatedRegistry.defaultVersion == registry.defaultVersion) { + logger.lifecycle( + "Not updating default version in {} because it is already the latest version: {}", + jsonFile.absolutePath, + updatedRegistry.defaultVersion ) + } else { + logger.lifecycle( + "Updating default version in {} to the latest version: {} (was {})", + jsonFile.absolutePath, + updatedRegistry.defaultVersion, + registry.defaultVersion + ) + } - val newVersions = versions.toMutableList() - for (updatedVersion in updatedVersions) { - val index = - newVersions.indexOfFirst { - it.version == version && it.os == updatedVersion.operatingSystem - } - if (index >= 0) { - val newVersion = - newVersions[index].copy( - size = updatedVersion.sizeInBytes, - sha512DigestHex = updatedVersion.sha512DigestHex, - ) - newVersions[index] = newVersion - } else { - val newVersion = - DataConnectExecutableVersionsRegistry.VersionInfo( - version = version, - os = updatedVersion.operatingSystem, - size = updatedVersion.sizeInBytes, - sha512DigestHex = updatedVersion.sha512DigestHex, + DataConnectExecutableVersionsRegistry.save(updatedRegistry, jsonFile) + } + + private data class CloudStorageVersionInfo( + val version: Version, + val operatingSystem: OperatingSystem, + val blob: Blob, + ) + + private fun Storage.getDataConnectExecutablesBucket(): Bucket { + val bucketName = "firemat-preview-drop" + logger.lifecycle("Finding all Data Connect executable versions in GCS bucket: {}", bucketName) + + return runCatching { get(bucketName) } + .onFailure { e -> + if ( + e is StorageException && + e.cause.let { + it is com.google.api.client.http.HttpResponseException && + (it.statusCode == 401 || it.statusCode == 403) + } + ) { + logger.error( + "ERROR: 401/403 error returned from Google Cloud Storage; " + + "try running \"gcloud auth application-default login\" and/or unsetting the " + + "GOOGLE_APPLICATION_CREDENTIALS environment variable to fix" ) - newVersions.add(newVersion) + } } + .getOrThrow() + ?: throw DataConnectGradleException("bvkxzp2esg", "GCS bucket not found: $bucketName") + } + + private fun Blob.toCloudStorageVersionInfoOrNull(): CloudStorageVersionInfo? { + logger.debug("[av7zhespw2] Found Data Connect executable file: {}", name) + val match = + fileNameRegex.matchEntire(name) + ?: run { + logger.debug( + "[p4vjjcp2kq] Ignoring Data Connect executable file: {} " + + "(does not match regex: {})", + name, + fileNameRegex + ) + return null + } + + val versionString = match.groups[2]?.value + val version = versionString?.toVersionOrNull(strict = false) + if (version === null) { + logger.info( + "Ignoring Data Connect executable file: {} " + + "(invalid version: {} (in match for regex {}))", + name, + versionString, + fileNameRegex + ) + return null } - return this.copy(versions = newVersions.toList()) - } + if (version < minVersion) { + logger.info( + "Ignoring Data Connect executable file: {} " + + "(version {} is less than the minimum version: {})", + name, + versionString, + minVersion + ) + return null + } - private fun download( - version: String, - operatingSystem: OperatingSystem, - outputDirectory: File - ): DownloadedFile { - val randomId = Random.nextAlphanumericString(length = 20) - val outputFile = - File(outputDirectory, "DataConnectToolkit_${version}_${operatingSystem}_$randomId") + if (version in invalidVersions) { + logger.info( + "Ignoring Data Connect executable file: {} (version {} is a known invalid version)", + name, + versionString + ) + return null + } - downloadDataConnectExecutable(version, operatingSystem, outputFile, execOperations) + val operatingSystem = + when (val operatingSystemString = match.groups[1]?.value) { + "linux" -> OperatingSystem.Linux + "macos" -> OperatingSystem.MacOS + "windows" -> OperatingSystem.Windows + else -> { + logger.info( + "WARNING: Ignoring Data Connect executable file: {} " + + "(unknown operating system name: {} (in match for regex {}))", + name, + operatingSystemString, + fileNameRegex + ) + return null + } + } - logger.info("Calculating SHA512 hash of file: {}", outputFile.absolutePath) - val fileInfo = FileInfo.forFile(outputFile) + return CloudStorageVersionInfo(version, operatingSystem, blob = this) + } + + private fun CloudStorageVersionInfo.toRegistryVersionInfo(workDirectory: File): VersionInfo { + val dateFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.LONG) - return DownloadedFile( - file = outputFile, - sizeInBytes = fileInfo.sizeInBytes, - sha512DigestHex = fileInfo.sha512DigestHex, + logger.lifecycle( + "Downloading version {} ({} bytes, created {})", + "$version-${operatingSystem.serializedValue}", + blob.size.toStringWithThousandsSeparator(), + dateFormatter.format(blob.createTimeOffsetDateTime.atZoneSameInstant(ZoneId.systemDefault())) ) + workDirectory.mkdirs() + val outputFile = File(workDirectory, Random.nextAlphanumericString(12)) + outputFile.outputStream().use { dest -> blob.downloadTo(dest) } + + val fileInfo = DataConnectExecutableDownloadTask.FileInfo.forFile(outputFile) + outputFile.delete() + check(fileInfo.sizeInBytes == blob.size) { + "fileInfo.sizeInBytes!=blob.size (${fileInfo.sizeInBytes}!=${blob.size}) and this should " + + "never happen; if it _does_ happen it _could_ indicate a compromised " + + "downloaded binary [y5967yd2cf]" + } + return VersionInfo(version, operatingSystem, fileInfo.sizeInBytes, fileInfo.sha512DigestHex) } - private data class DownloadedFile( - val file: File, - val sizeInBytes: Long, - val sha512DigestHex: String, - ) + private companion object { + + val versionInfoComparator = + compareBy { it.version }.thenByDescending { it.os.serializedValue } + + val cloudStorageVersionInfoComparator = + compareBy { it.version } + .thenByDescending { it.operatingSystem.serializedValue } + + @JvmName("toLogStringCloudStorageVersionInfo") + fun Iterable.toLogString(): String = joinToString { + "${it.version}-${it.operatingSystem.serializedValue}" + } + + @JvmName("toLogStringVersionInfo") + fun Iterable.toLogString(): String = joinToString { + "${it.version}-${it.os.serializedValue}" + } + + val invalidVersions = setOf("1.15.0".toVersion()) + val minVersion = "1.3.4".toVersion() + val fileNameRegex = ".*dataconnect-emulator-([^-]+)-v(.*)".toRegex() - enum class UpdateMode { - Overwrite, - Update + /** + * Creates and returns a new list that contains all elements of the receiving [Iterable] that + * are not in the given registry. + */ + private fun Iterable.filterNotIn( + registry: DataConnectExecutableVersionsRegistry.Root + ): List = filterNot { cloudStorageVersion -> + registry.versions.any { + it.version == cloudStorageVersion.version && it.os == cloudStorageVersion.operatingSystem + } + } + + private fun DataConnectExecutableVersionsRegistry.Root.updatedWith( + updatedVersions: Iterable + ): DataConnectExecutableVersionsRegistry.Root { + val allVersions = buildList { + addAll(versions) + addAll(updatedVersions) + sortWith(versionInfoComparator) + } + return copy(defaultVersion = allVersions.maxOf { it.version }, versions = allVersions) + } } } diff --git a/firebase-dataconnect/scripts/missingversions.py b/firebase-dataconnect/scripts/missingversions.py deleted file mode 100755 index 80ee9ecfe58..00000000000 --- a/firebase-dataconnect/scripts/missingversions.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run this script in the root directory of this Git repository to -# determine the versions of the Data Connect Toolkit that are missing -# from the DataConnectExecutableVersions.json file. The final output -# of this script will be the gradle command to run to update the json -# file. -# -# Make sure to run "pip install packaging" before running this script. - -import json -import os -from packaging.version import Version -import re -import subprocess -import tempfile - -regex = re.compile(r".*dataconnect-emulator-linux-v(\d+\.\d+\.\d+)") -json_path = os.path.abspath("firebase-dataconnect/gradleplugin/plugin/src/main/resources/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableVersions.json") -min_version = Version("1.3.4") -bucket = "gs://firemat-preview-drop/emulator/" - -args = ["gsutil", "ls", "-r", bucket] -print("Getting versions by running: " + subprocess.list2cmdline(args)) -with tempfile.TemporaryFile() as f: - subprocess.check_call(args, stdout=f) - f.seek(0) - filenames = f.read().decode("utf8", errors="strict").splitlines() - -filename_matches = [regex.fullmatch(filename) for filename in filenames] -versions_set = set(match.group(1) for match in filename_matches if match is not None) -all_versions = sorted(versions_set, key=Version) -versions = [version for version in all_versions if Version(version) >= min_version] - -try: - invalid_version_index = versions.index("1.15.0") -except ValueError: - pass -else: - versions.pop(invalid_version_index) - -print(f"Found {len(versions)} versions greater than {min_version}: {versions!r}") -print() - -with open(json_path, "rb") as f: - known_versions_map = json.load(f) -known_versions_set = frozenset(version_info["version"] for version_info in known_versions_map["versions"]) -known_versions = sorted(known_versions_set, key=Version) -print(f"Found {len(known_versions)} versions in {os.path.basename(json_path)}: {known_versions!r}") -print() - -missing_versions = [version for version in versions if version not in known_versions] -print(f"Found {len(missing_versions)} missing versions in {os.path.basename(json_path)}: {missing_versions!r}") -print() - -print(f"Run this gradle command to update {json_path}:") -print(f"./gradlew :firebase-dataconnect:connectors:updateJson -Pversions={",".join(missing_versions)} -PdefaultVersion={versions[-1]}") diff --git a/firebase-dataconnect/scripts/update_versions_json.sh b/firebase-dataconnect/scripts/update_versions_json.sh new file mode 100755 index 00000000000..ff04bbc20f3 --- /dev/null +++ b/firebase-dataconnect/scripts/update_versions_json.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +PROJECT_ROOT_DIR="$(dirname "$0")/../.." +readonly PROJECT_ROOT_DIR + +readonly args=( + "${PROJECT_ROOT_DIR}/gradlew" + "-p" + "${PROJECT_ROOT_DIR}" + "--configure-on-demand" + "$@" + ":firebase-dataconnect:updateJson" +) + +echo "${args[*]}" +exec "${args[@]}"