Added utilities to debug protobuf conformance tests (#453)

Author: Mr3zee

.setup_protoscope.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+#
+# Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+#
+
+echo "Installing protoscope"
+
+brew install go
+go install github.com/protocolbuffers/protoscope/cmd/protoscope...@latest
+
+PROTOSCOPE_PATH=~/go/bin/protoscope
+if [ -f "$PROTOSCOPE_PATH" ]; then
+  if grep -q "protoscope_path=" local.properties; then
+    sed -i '' "s|protoscope_path=.*|protoscope_path=$PROTOSCOPE_PATH|" local.properties
+  else
+    echo "protoscope_path=$PROTOSCOPE_PATH" >> local.properties
+  fi
+else
+  echo "Error: protoscope not found at $PROTOSCOPE_PATH"
+  exit 1
+fi

docs/environment.md

@@ -363,6 +363,39 @@ TODO: write a guide about kRPC API check tests.
 
 Use `./publishLocal.sh` script. All artifacts will be in the local directory of `<REPO_ROOT>/build/repo/` .
 
+## How to debug tests/protobuf-conformance
+
+Prerequisite (Only macOS for now): 
+```bash
+./setup_protoscope.sh
+```
+It will install the `protoscope` utility: https://github.com/protocolbuffers/protoscope
+
+Now you can run tests:
+```bash
+gradle :tests:protobuf-conformance:runConformanceTest -Pconformance.test='<test_name>'
+```
+
+In the [manual](../tests/protobuf-conformance/build/protobuf-conformance/manual) directory 
+you will find files: 
+- dump_conformance_input.bin.txt - decoded ConformanceRequest protobuf message
+- dump_conformance_output.bin.txt - decoded ConformanceResponse protobuf message
+- dump_payload_input.bin.txt - decoded ConformanceRequest.payload (if protobuf)
+- dump_payload_output.bin.txt - decoded ConformanceResponse.result (if protobuf)
+
+IMPORTANT: `protoscope` only works with proto3 and not 'editions' messages. 
+For proto2 and editions this won't work.
+
+To debug tests, 
+use [ConformanceClient.kt](../tests/protobuf-conformance/src/main/kotlin/kotlinx/rpc/protoc/gen/test/ConformanceClient.kt)
+and this command:
+```bash
+gradle :tests:protobuf-conformance:runConformanceTest -Pconformance.test.debug='true' -Pconformance.test='<test_name>'
+```
+
+Then use IntelliJ 'Attach to Process' feature to attach to the process on port 5005.
+(kill the process if port is already in use: `kill $(lsof -t -i:5005)`)
+
 ## Troubleshooting
 
 Nothing works? Well, you are onto a journey!

gradle-conventions/src/main/kotlin/util/other/localProperties.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package util.other
+
+import org.gradle.api.Project
+import org.gradle.internal.extensions.core.extra
+import org.gradle.kotlin.dsl.provideDelegate
+import java.util.Properties
+import java.util.concurrent.atomic.AtomicReference
+import kotlin.io.path.Path
+import kotlin.io.path.inputStream
+
+private val ref = AtomicReference<Properties>()
+
+fun Project.localProperties(): Properties {
+    if (ref.get() == null) {
+        ref.compareAndSet(null, Properties().apply {
+            val globalRootDir: String by extra
+
+            load(Path(globalRootDir, "local.properties").inputStream())
+        })
+    }
+
+    return ref.get()
+}

gradle-conventions/src/main/kotlin/util/tasks/protobufConformanceUpdate.kt

@@ -6,10 +6,14 @@ package util.tasks
 
 import org.gradle.api.DefaultTask
 import org.gradle.api.Project
+import org.gradle.api.file.FileTree
+import org.gradle.api.provider.ListProperty
 import org.gradle.api.provider.Property
 import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.Exec
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFile
+import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.TaskAction
 import org.gradle.kotlin.dsl.accessors.runtime.addExternalModuleDependencyTo
@@ -21,8 +25,40 @@ import util.other.libs
 import java.io.File
 
 const val CONFORMANCE_TEST_RUNNER_CONFIGURATION = "conformanceTestRunner"
+const val PROTOC_TESTING_CONFIGURATION = "protoc_internalTesting"
 const val UNZIP_PROTOBUF_CONFORMANCE_TASK = "unzipProtobufConformance"
 const val WRITE_CONFORMANCE_EXECUTABLE_PATH_TASK = "writeConformanceExecutablePath"
+const val PAYLOAD_PB = "payload.pb"
+const val CONFORMANCE_PB = "conformance.pb"
+
+private fun Project.getBinFrom(configuration: String): File {
+    return configurations.getByName(configuration).map {
+        zipTree(it).matching { include("bin/**") }.files.first()
+    }.single()
+}
+
+private fun Project.getIncludeFrom(configuration: String): List<FileTree> {
+    return configurations.getByName(configuration).map {
+        zipTree(it).matching { include("include/**") }
+    }
+}
+
+private fun List<String>.commonPrefix(): String {
+    return fold(first()) { acc, s -> acc.commonPrefixWith(s) }
+}
+
+private fun Project.pbFile(name: String): File {
+    return layout.buildDirectory.get()
+        .dir("protobuf-conformance")
+        .file(name)
+        .asFile
+        .apply {
+            if (!exists()) {
+                parentFile.mkdirs()
+                createNewFile()
+            }
+        }
+}
 
 abstract class ConformanceExecutablePathWriter : DefaultTask() {
     @get:Input
@@ -64,6 +100,48 @@ abstract class ConformanceExecutablePathWriter : DefaultTask() {
     }
 }
 
+abstract class GenerateConformanceFileDescriptorSet : Exec() {
+    @get:InputFiles
+    abstract val wktFilesCollection: ListProperty<File>
+
+    @get:InputFiles
+    abstract val conformanceFilesCollection: ListProperty<File>
+
+    @get:InputFile
+    abstract val bin: Property<File>
+
+    @get:OutputFile
+    abstract val outputFile: Property<File>
+
+    @TaskAction
+    fun generate() {
+        val wktFiles = wktFilesCollection.get().map { it.absolutePath }
+        val conformanceFiles = conformanceFilesCollection.get().map { it.absolutePath }
+
+        val wktProtoPath = if (wktFiles.isEmpty()) {
+            emptyList()
+        } else {
+            listOf("--proto_path=${wktFiles.commonPrefix().substringBefore("/google/protobuf/")}")
+        }
+
+        val conformanceIncludeDir = conformanceFiles
+            .commonPrefix()
+            .substringBefore("/google/protobuf/")
+            .substringBefore("/conformance/")
+
+        val conformanceProtoPath = "--proto_path=$conformanceIncludeDir"
+
+        commandLine(
+            bin.get().absolutePath,
+            *wktProtoPath.toTypedArray(),
+            conformanceProtoPath,
+            "-o", outputFile.get().absolutePath,
+            *wktFiles.toTypedArray(),
+            *conformanceFiles.toTypedArray(),
+        )
+    }
+}
+
 fun Project.setupProtobufConformanceResources() {
     val os = System.getProperty("os.name").lowercase()
     val osPart = when {
@@ -79,7 +157,7 @@ fun Project.setupProtobufConformanceResources() {
 
     // https://stackoverflow.com/questions/23023069/gradle-download-and-unzip-file-from-url
     repositories.ivy {
-        name = "protobuf-conformance-github"
+        name = "github"
         url = uri("https://github.com")
 
         patternLayout {
@@ -93,6 +171,7 @@ fun Project.setupProtobufConformanceResources() {
     }
 
     configurations.create(CONFORMANCE_TEST_RUNNER_CONFIGURATION)
+    configurations.create(PROTOC_TESTING_CONFIGURATION)
 
     // https://docs.gradle.org/current/javadoc/org/gradle/api/artifacts/dsl/DependencyHandler.html
     dependencies {
@@ -115,10 +194,28 @@ fun Project.setupProtobufConformanceResources() {
         }
     }
 
+    dependencies {
+        addExternalModuleDependencyTo(
+            this,
+            PROTOC_TESTING_CONFIGURATION,
+            group = "protocolbuffers",
+            name = "protobuf",
+            version = libs.versions.protobuf.asProvider().get().substringAfter("."),
+            classifier = null,
+            ext = null,
+            configuration = null,
+        ) {
+            artifact {
+                name = "protoc"
+                type = "zip"
+                extension = "zip"
+                classifier = "$osPart-$archPart"
+            }
+        }
+    }
+
     val unzipProtobufConformance = tasks.register<Copy>(UNZIP_PROTOBUF_CONFORMANCE_TASK) {
-        from(configurations.getByName(CONFORMANCE_TEST_RUNNER_CONFIGURATION).map {
-            zipTree(it).matching { include("include/**") }
-        })
+        from(getIncludeFrom(CONFORMANCE_TEST_RUNNER_CONFIGURATION))
 
         val destDir = project.layout.projectDirectory
             .dir("src")
@@ -140,6 +237,34 @@ fun Project.setupProtobufConformanceResources() {
         }
     }
 
+    tasks.register<GenerateConformanceFileDescriptorSet>("generateConformanceFileDescriptorSet_conformance") {
+        wktFilesCollection.set(emptyList())
+
+        val conformanceFiles = project.getIncludeFrom(CONFORMANCE_TEST_RUNNER_CONFIGURATION).flatMap { it.files }
+            .filter { it.name == "conformance.proto" }
+
+        conformanceFilesCollection.set(conformanceFiles)
+
+        bin.set(getBinFrom(PROTOC_TESTING_CONFIGURATION))
+
+        outputFile.set(project.pbFile(CONFORMANCE_PB))
+    }
+
+    tasks.register<GenerateConformanceFileDescriptorSet>("generateConformanceFileDescriptorSet_payload") {
+        val wktFiles = project.getIncludeFrom(PROTOC_TESTING_CONFIGURATION).flatMap { it.files }
+        wktFilesCollection.set(wktFiles)
+
+        // editions are not supported in protoscope and proto2 fails
+        val conformanceFiles = project.getIncludeFrom(CONFORMANCE_TEST_RUNNER_CONFIGURATION).flatMap { it.files }
+            .filter { it.name == "test_messages_proto3.proto" }
+
+        conformanceFilesCollection.set(conformanceFiles)
+
+        bin.set(getBinFrom(PROTOC_TESTING_CONFIGURATION))
+
+        outputFile.set(project.pbFile(PAYLOAD_PB))
+    }
+
     val writeConformanceExecutablePath =
         tasks.register<ConformanceExecutablePathWriter>(WRITE_CONFORMANCE_EXECUTABLE_PATH_TASK) {
             outputDir.set(
@@ -156,11 +281,7 @@ fun Project.setupProtobufConformanceResources() {
                     .asFile
             )
 
-            executable.set(
-                configurations.getByName(CONFORMANCE_TEST_RUNNER_CONFIGURATION).map {
-                    zipTree(it).matching { include("bin/**") }.files.first()
-                }.single()
-            )
+            executable.set(getBinFrom(CONFORMANCE_TEST_RUNNER_CONFIGURATION))
 
             destination.set(
                 project.layout.buildDirectory.get()

tests/protobuf-conformance/build.gradle.kts

@@ -8,6 +8,10 @@ import kotlinx.rpc.buf.tasks.BufGenerateTask
 import kotlinx.rpc.internal.InternalRpcApi
 import kotlinx.rpc.internal.configureLocalProtocGenDevelopmentDependency
 import org.jetbrains.kotlin.gradle.dsl.ExplicitApiMode
+import util.other.localProperties
+import util.tasks.CONFORMANCE_PB
+import util.tasks.GenerateConformanceFileDescriptorSet
+import util.tasks.PAYLOAD_PB
 import util.tasks.setupProtobufConformanceResources
 
 plugins {
@@ -86,6 +90,58 @@ val generateConformanceTests = tasks.register<JavaExec>("generateConformanceTest
     mainClass.set("kotlinx.rpc.protoc.gen.test.GenerateConformanceTestsKt")
 }
 
+val conformanceTest = properties.getOrDefault("conformance.test", "").toString()
+val conformanceTestDebug = properties.getOrDefault("conformance.test.debug", "false").toString().toBooleanStrictOrNull() ?: false
+
+val generateConformanceFileDescriptorSet = tasks
+    .withType<GenerateConformanceFileDescriptorSet>()
+
+tasks.register<JavaExec>("runConformanceTest") {
+    classpath = sourceSets.main.get().runtimeClasspath
+
+    dependsOn(mockClientJar)
+    dependsOn(tasks.named("bufGenerateMain"))
+    dependsOn(generateConformanceFileDescriptorSet)
+
+    args = listOfNotNull(
+        mockClientJar.get().archiveFile.get().asFile.absolutePath,
+        conformanceTest,
+        if (conformanceTestDebug) "--debug" else null
+    )
+
+    mainClass.set("kotlinx.rpc.protoc.gen.test.RunConformanceTestKt")
+
+    val protoscope = localProperties().getProperty("protoscope_path")
+        ?: throw GradleException("protoscope_path property is not set. Run ./setup_protoscope.sh")
+
+    environment("PROTOSCOPE_PATH", protoscope)
+
+    val pbFiles = generateConformanceFileDescriptorSet.map {
+        it.outputFile.get()
+    }
+
+    environment(
+        "CONFORMANCE_PB_PATH",
+        pbFiles.single { it.name == CONFORMANCE_PB }.absolutePath
+    )
+    environment(
+        "TEST_ALL_TYPES_PROTO3_PB_PATH",
+        pbFiles.single { it.name == PAYLOAD_PB }.absolutePath
+    )
+
+    doFirst {
+        if (!File(protoscope).exists()) {
+            throw GradleException(
+                """
+                    Protoscope is not found. Use the following command to install it: 
+                    
+                    $ brew install go
+                    $ go install github.com/protocolbuffers/protoscope/cmd/protoscope...@latest
+                """.trimIndent()
+            )
+        }
+    }
+}
 
 tasks.test {
     environment("MOCK_CLIENT_JAR", mockClientJar.get().archiveFile.get().asFile.absolutePath)