Set up protobuf conformance tests

.github/workflows/protobuf-conformance.yml

@@ -0,0 +1,27 @@
+name: Verify Protobuf Conformance is Up-to-Date
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  verify-platforms-table:
+    name: Run Verification
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Sources
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+      - name: Run Protobuf Conformance Test Generation
+        run: ./gradlew tests:protobuf-conformance:bufGenerateTest --info --stacktrace
+      - name: Check if protobuf-conformance test is up-to-date
+        run: |
+          if [[ -n "$(git status --porcelain | grep tests/protobuf-conformance/)" ]]; then
+            echo "Protobuf conformance test is not up to date. Please run './gradlew tests:protobuf-conformance:bufGenerateTest' and commit changes"
+            exit 1
+          fi

.github/workflows/protobuf-well-known-types.yml

@@ -0,0 +1,27 @@
+name: Verify Protobuf Well-Known Types are Up-to-Date
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  verify-platforms-table:
+    name: Run Verification
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Sources
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+      - name: Run Protobuf Conformance Test Generation
+        run: ./gradlew :protobuf:protobuf-core:bufGenerateCommonMain --info --stacktrace
+      - name: Check if Well-Known Types are up-to-date
+        run: |
+          if [[ -n "$(git status --porcelain | grep protobuf/protobuf-core/)" ]]; then
+            echo "Well-Known Types are not up-to-date. Please run './gradlew :protobuf:protobuf-core:bufGenerateCommonMain' and commit changes"
+            exit 1
+          fi

gradle-conventions/src/main/kotlin/util/tasks/protobufConformanceUpdate.kt

@@ -0,0 +1,183 @@
+/*
+ * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package util.tasks
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.Project
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFile
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.TaskAction
+import org.gradle.kotlin.dsl.accessors.runtime.addExternalModuleDependencyTo
+import org.gradle.kotlin.dsl.dependencies
+import org.gradle.kotlin.dsl.register
+import org.gradle.kotlin.dsl.the
+import org.jetbrains.kotlin.gradle.dsl.KotlinJvmExtension
+import util.other.libs
+import java.io.File
+
+const val CONFORMANCE_TEST_RUNNER_CONFIGURATION = "conformanceTestRunner"
+const val UNZIP_PROTOBUF_CONFORMANCE_TASK = "unzipProtobufConformance"
+const val WRITE_CONFORMANCE_EXECUTABLE_PATH_TASK = "writeConformanceExecutablePath"
+
+abstract class ConformanceExecutablePathWriter : DefaultTask() {
+    @get:Input
+    abstract val outputDir: Property<File>
+
+    @get:Input
+    abstract val kotlinTestDir: Property<File>
+
+    @get:InputFile
+    abstract val executable: Property<File>
+
+    @get:OutputFile
+    abstract val destination: Property<File>
+
+    @TaskAction
+    fun action() {
+        val dest = destination.get()
+        if (dest.exists()) {
+            dest.delete()
+        }
+
+        val parent = dest.parentFile
+        if (!parent.exists()) {
+            parent.mkdirs()
+        }
+
+        dest.writeText(
+            """
+            // file generated by ConformanceExecutablePathWriter task
+
+            const val CONFORMANCE_EXECUTABLE_PATH: String = "${executable.get().absolutePath}"
+
+            const val CONFORMANCE_OUTPUT_DIR: String = "${outputDir.get().absolutePath}"
+
+            const val KOTLIN_TEST_DIR: String = "${kotlinTestDir.get().absolutePath}"
+
+        """.trimIndent()
+        )
+    }
+}
+
+fun Project.setupProtobufConformanceResources() {
+    val os = System.getProperty("os.name").lowercase()
+    val osPart = when {
+        os.startsWith("linux") -> "linux"
+        os.startsWith("mac") -> "osx"
+        else -> error("unsupported os for protobuf-conformance tests: $os")
+    }
+
+    val archPart = when (val arch = System.getProperty("os.arch").lowercase()) {
+        in setOf("x86_64", "amd64", "aarch64") -> "x86_64"
+        else -> error("unsupported arch for protobuf-conformance tests: $arch")
+    }
+
+    // https://stackoverflow.com/questions/23023069/gradle-download-and-unzip-file-from-url
+    repositories.ivy {
+        name = "protobuf-conformance-github"
+        url = uri("https://github.com")
+
+        patternLayout {
+            // https://github.com/bufbuild/protobuf-conformance/releases/download/v${version}/conformance_test_runner-${version}-${build}.zip
+            artifact("[organisation]/[module]/releases/download/v[revision]/[artifact]-[revision]-[classifier].[ext]")
+        }
+
+        // This is required in Gradle 6.0+ as metadata file (ivy.xml)
+        // is mandatory. Docs linked below this code section
+        metadataSources { artifact() }
+    }
+
+    configurations.create(CONFORMANCE_TEST_RUNNER_CONFIGURATION)
+
+    // https://docs.gradle.org/current/javadoc/org/gradle/api/artifacts/dsl/DependencyHandler.html
+    dependencies {
+        addExternalModuleDependencyTo(
+            this,
+            CONFORMANCE_TEST_RUNNER_CONFIGURATION,
+            group = "bufbuild",
+            name = "protobuf-conformance",
+            version = libs.versions.protobuf.asProvider().get().substringAfter("."),
+            classifier = null,
+            ext = null,
+            configuration = null,
+        ) {
+            artifact {
+                name = "conformance_test_runner"
+                type = "zip"
+                extension = "zip"
+                classifier = "$osPart-$archPart"
+            }
+        }
+    }
+
+    val unzipProtobufConformance = tasks.register<Copy>(UNZIP_PROTOBUF_CONFORMANCE_TASK) {
+        from(configurations.getByName(CONFORMANCE_TEST_RUNNER_CONFIGURATION).map {
+            zipTree(it).matching { include("include/**") }
+        })
+
+        val destDir = project.layout.projectDirectory
+            .dir("src")
+            .dir("main")
+            .dir("proto")
+
+        into(destDir)
+
+        eachFile {
+            if (path.startsWith("include/")) {
+                path = "protobuf-conformance/${path.removePrefix("include/")}"
+            }
+        }
+
+        includeEmptyDirs = false
+
+        doFirst {
+            destDir.asFile.deleteRecursively()
+        }
+    }
+
+    val writeConformanceExecutablePath =
+        tasks.register<ConformanceExecutablePathWriter>(WRITE_CONFORMANCE_EXECUTABLE_PATH_TASK) {
+            outputDir.set(
+                project.layout.buildDirectory.get()
+                    .dir("protobuf-conformance")
+                    .asFile
+            )
+
+            kotlinTestDir.set(
+                project.layout.projectDirectory
+                    .dir("src")
+                    .dir("test")
+                    .dir("kotlin")
+                    .asFile
+            )
+
+            executable.set(
+                configurations.getByName(CONFORMANCE_TEST_RUNNER_CONFIGURATION).map {
+                    zipTree(it).matching { include("bin/**") }.files.first()
+                }.single()
+            )
+
+            destination.set(
+                project.layout.buildDirectory.get()
+                    .dir("generated")
+                    .dir("protobuf-conformance")
+                    .file("executable-paths.kt")
+                    .asFile
+            )
+        }
+
+    tasks.matching { it.name == "processMainProtoFiles" || it.name == "processTestImportProtoFiles" }.all {
+        dependsOn(unzipProtobufConformance)
+    }
+
+    the<KotlinJvmExtension>().apply {
+        sourceSets.getByName("main") {
+            kotlin.srcDir(writeConformanceExecutablePath.map { it.destination.get().parentFile })
+        }
+    }
+}

gradle-plugin/api/gradle-plugin.api

@@ -39,11 +39,7 @@ public class kotlinx/rpc/RpcStrictModeExtension {
 
 public final class kotlinx/rpc/VersionsKt {
 	public static final field BUF_TOOL_VERSION Ljava/lang/String;
-	public static final field GRPC_KOTLIN_VERSION Ljava/lang/String;
-	public static final field GRPC_VERSION Ljava/lang/String;
 	public static final field LIBRARY_VERSION Ljava/lang/String;
-	public static final field PLUGIN_VERSION Ljava/lang/String;
-	public static final field PROTOBUF_VERSION Ljava/lang/String;
 }
 
 public class kotlinx/rpc/buf/BufExtension {
@@ -179,17 +175,15 @@ public final class kotlinx/rpc/protoc/ProcessProtoFilesKt$inlined$sam$i$org_grad
 
 public abstract interface class kotlinx/rpc/protoc/ProtoSourceSet {
 	public abstract fun getName ()Ljava/lang/String;
+	public abstract fun getPlugins ()Lorg/gradle/api/NamedDomainObjectContainer;
 	public abstract fun getProto ()Lorg/gradle/api/file/SourceDirectorySet;
+	public abstract fun plugins (Lorg/gradle/api/Action;)V
 	public fun proto (Lorg/gradle/api/Action;)V
-	public abstract fun protocPlugin (Lkotlinx/rpc/protoc/ProtocPlugin;)V
-	public abstract fun protocPlugin (Lorg/gradle/api/NamedDomainObjectProvider;)V
 }
 
 public abstract interface class kotlinx/rpc/protoc/ProtocExtension {
 	public abstract fun buf (Lorg/gradle/api/Action;)V
 	public abstract fun getBuf ()Lkotlinx/rpc/buf/BufExtension;
-	public abstract fun getPlugins ()Lorg/gradle/api/NamedDomainObjectContainer;
-	public abstract fun plugins (Lorg/gradle/api/Action;)V
 }
 
 public class kotlinx/rpc/protoc/ProtocPlugin {

gradle-plugin/build.gradle.kts

@@ -83,9 +83,6 @@ gradlePlugin {
 
 abstract class GeneratePluginVersionsTask @Inject constructor(
     @get:Input val libraryVersion: String,
-    @get:Input val protobufVersion: String,
-    @get:Input val grpcVersion: String,
-    @get:Input val grpcKotlinVersion: String,
     @get:Input val bufToolVersion: String,
     @get:OutputDirectory val sourcesDir: File
 ) : DefaultTask() {
@@ -104,24 +101,6 @@ package kotlinx.rpc
  */
 public const val LIBRARY_VERSION: String = "$libraryVersion"
 
-@Deprecated("Use kotlinx.rpc.LIBRARY_VERSION instead", ReplaceWith("kotlinx.rpc.LIBRARY_VERSION"))
-public const val PLUGIN_VERSION: String = LIBRARY_VERSION
-
-/**
- * The version of the protobuf library.
- */
-public const val PROTOBUF_VERSION: String = "$protobufVersion"
-
-/**
- * The version of the grpc java library.
- */
-public const val GRPC_VERSION: String = "$grpcVersion"
-
-/**
- * The version of the grpc kotlin library.
- */
-public const val GRPC_KOTLIN_VERSION: String = "$grpcKotlinVersion"
-
 /**
  * The version of the buf tool used to generate protobuf.
  */
@@ -141,18 +120,12 @@ val sourcesDir = File(project.layout.buildDirectory.asFile.get(), "generated-sou
 val generatePluginVersionsTask = tasks.register<GeneratePluginVersionsTask>(
     GeneratePluginVersionsTask.NAME,
     version.toString(),
-    libs.versions.protobuf.asProvider().get(),
-    libs.versions.grpc.asProvider().get(),
-    libs.versions.grpc.kotlin.get(),
     libs.versions.buf.tool.get(),
     sourcesDir,
 )
 
 abstract class GenerateTestVersionTask @Inject constructor(
     @get:Input val kotlinVersion: String,
-    @get:Input val protobufVersion: String,
-    @get:Input val grpcVersion: String,
-    @get:Input val grpcKotlinVersion: String,
     @get:Input val buildRepo: String,
     @get:OutputDirectory val sourcesDir: File
 ) : DefaultTask() {
@@ -170,11 +143,6 @@ const val KOTLIN_VERSION: String = "$kotlinVersion"
 
 const val BUILD_REPO: String = "$buildRepo"
 
-// can't use from generatePluginVersionsTask bacause Gradle messes up caches
-const val TEST_PROTOBUF_VERSION: String = "$protobufVersion"
-const val TEST_GRPC_VERSION: String = "$grpcVersion"
-const val TEST_GRPC_KOTLIN_VERSION: String = "$grpcKotlinVersion"
-
 """.trimIndent()
         )
     }
@@ -191,9 +159,6 @@ val globalRootDir: String by extra
 val generateTestVersionsTask = tasks.register<GenerateTestVersionTask>(
     GenerateTestVersionTask.NAME,
     libs.versions.kotlin.lang.get(),
-    libs.versions.protobuf.asProvider().get(),
-    libs.versions.grpc.asProvider().get(),
-    libs.versions.grpc.kotlin.get(),
     File(globalRootDir).resolve("build/repo").absolutePath,
     testSourcesDir,
 )

gradle-plugin/src/main/kotlin/kotlinx/rpc/Extensions.kt

@@ -19,7 +19,8 @@ import org.gradle.kotlin.dsl.property
 import java.util.concurrent.atomic.AtomicBoolean
 import javax.inject.Inject
 
-internal fun Project.rpcExtension(): RpcExtension = extensions.findByType<RpcExtension>() ?: RpcExtension(objects, this)
+internal fun Project.rpcExtension(): RpcExtension = extensions.findByType<RpcExtension>()
+    ?: error("Rpc extension not found. Please apply the plugin to the project")
 
 public open class RpcExtension @Inject constructor(objects: ObjectFactory, private val project: Project) {
     /**
@@ -51,6 +52,10 @@ public open class RpcExtension @Inject constructor(objects: ObjectFactory, priva
      * Protoc settings.
      */
     public val protoc: ProtocExtension by lazy {
+        if (protocApplied.get()) {
+            error("Illegal access to protoc extension during DefaultProtocExtension.init")
+        }
+
         protocApplied.set(true)
         objects.newInstance<DefaultProtocExtension>()
     }

gradle-plugin/src/main/kotlin/kotlinx/rpc/buf/tasks/BufExecTask.kt

@@ -117,11 +117,11 @@ internal fun <T : BufExecTask> Project.registerBufExecTask(
     bufExecutable.set(executableConfiguration.singleFile)
     this.workingDir.set(workingDir)
 
-    val buf = project.rpcExtension().protoc.buf
-    configFile.set(buf.configFile)
-    logFormat.set(buf.logFormat)
-    bufTimeoutInWholeSeconds.set(buf.timeout.map { it.inWholeSeconds })
-    debug.set(project.gradle.startParameter.logLevel == LogLevel.DEBUG)
+    val buf = provider { rpcExtension().protoc.buf }
+    configFile.set(buf.flatMap { it.configFile })
+    logFormat.set(buf.flatMap { it.logFormat })
+    bufTimeoutInWholeSeconds.set(buf.flatMap { it.timeout.map { duration -> duration.inWholeSeconds } })
+    debug.set(gradle.startParameter.logLevel == LogLevel.DEBUG)
 
     configuration()
 }