Skip to content

Commit f7f386d

Browse files
Mr3zeeMr3zee
committed
Set up protobuf conformance tests:
- Added runner exe - Added conformance protos and generated code - Fixed low-hanging gradle and protoc-gen issues
1 parent 7d56905 commit f7f386d

File tree

32 files changed

+12556
-106
lines changed

32 files changed

+12556
-106
lines changed

‎.github/workflows/protobuf-conformance.yml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Verify Protobuf Conformance is Up-to-Date
2+
3+
on:
4+
pull_request:
5+
6+
permissions:
7+
contents: read
8+
9+
jobs:
10+
verify-platforms-table:
11+
name: Run Verification
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout Sources
15+
uses: actions/checkout@v4
16+
with:
17+
fetch-depth: 0
18+
- name: Setup Gradle
19+
uses: gradle/actions/setup-gradle@v4
20+
- name: Run Protobuf Conformance Test Generation
21+
run: ./gradlew tests:protobuf-conformance:bufGenerateTest --info --stacktrace
22+
- name: Check if protobuf-conformance test is up-to-date
23+
run: |
24+
if [[ -n "$(git status --porcelain | grep tests/protobuf-conformance/)" ]]; then
25+
echo "Protobuf conformance test is not up to date. Please run './gradlew tests:protobuf-conformance:bufGenerateTest' and commit changes"
26+
exit 1
27+
fi

‎gradle-conventions/src/main/kotlin/util/tasks/protobufConformanceUpdate.kt

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/*
2+
* Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
package util.tasks
6+
7+
import org.gradle.api.DefaultTask
8+
import org.gradle.api.Project
9+
import org.gradle.api.provider.Property
10+
import org.gradle.api.tasks.Copy
11+
import org.gradle.api.tasks.InputFile
12+
import org.gradle.api.tasks.OutputFile
13+
import org.gradle.api.tasks.TaskAction
14+
import org.gradle.kotlin.dsl.accessors.runtime.addExternalModuleDependencyTo
15+
import org.gradle.kotlin.dsl.dependencies
16+
import org.gradle.kotlin.dsl.register
17+
import org.gradle.kotlin.dsl.the
18+
import org.jetbrains.kotlin.gradle.dsl.KotlinJvmExtension
19+
import util.other.libs
20+
import java.io.File
21+
22+
const val CONFORMANCE_TEST_RUNNER_CONFIGURATION = "conformanceTestRunner"
23+
const val UNZIP_PROTOBUF_CONFORMANCE_TASK = "unzipProtobufConformance"
24+
const val WRITE_CONFORMANCE_EXECUTABLE_PATH_TASK = "writeConformanceExecutablePath"
25+
26+
abstract class ConformanceExecutablePathWriter : DefaultTask() {
27+
@get:InputFile
28+
abstract val executable: Property<File>
29+
30+
@get:OutputFile
31+
abstract val destination: Property<File>
32+
33+
@TaskAction
34+
fun action() {
35+
val dest = destination.get()
36+
if (dest.exists()) {
37+
dest.delete()
38+
}
39+
40+
val parent = dest.parentFile
41+
if (!parent.exists()) {
42+
parent.mkdirs()
43+
}
44+
45+
dest.writeText(
46+
"""
47+
// file generated by ConformanceExecutablePathWriter task
48+
49+
const val CONFORMANCE_EXECUTABLE_PATH: String = "${executable.get().absolutePath}"
50+
51+
""".trimIndent()
52+
)
53+
}
54+
}
55+
56+
fun Project.setupProtobufConformanceResources() {
57+
val os = System.getProperty("os.name").lowercase()
58+
val osPart = when {
59+
os.startsWith("linux") -> "linux"
60+
os.startsWith("mac") -> "osx"
61+
else -> error("unsupported os for protobuf-conformance tests: $os")
62+
}
63+
64+
val archPart = when (val arch = System.getProperty("os.arch").lowercase()) {
65+
in setOf("x86_64", "amd64", "aarch64") -> "x86_64"
66+
else -> error("unsupported arch for protobuf-conformance tests: $arch")
67+
}
68+
69+
// https://stackoverflow.com/questions/23023069/gradle-download-and-unzip-file-from-url
70+
repositories.ivy {
71+
name = "protobuf-conformance-github"
72+
url = uri("https://github.com")
73+
74+
patternLayout {
75+
// https://github.com/bufbuild/protobuf-conformance/releases/download/v${version}/conformance_test_runner-${version}-${build}.zip
76+
artifact("[organisation]/[module]/releases/download/v[revision]/[artifact]-[revision]-[classifier].[ext]")
77+
}
78+
79+
// This is required in Gradle 6.0+ as metadata file (ivy.xml)
80+
// is mandatory. Docs linked below this code section
81+
metadataSources { artifact() }
82+
}
83+
84+
configurations.create(CONFORMANCE_TEST_RUNNER_CONFIGURATION)
85+
86+
// https://docs.gradle.org/current/javadoc/org/gradle/api/artifacts/dsl/DependencyHandler.html
87+
dependencies {
88+
addExternalModuleDependencyTo(
89+
this,
90+
CONFORMANCE_TEST_RUNNER_CONFIGURATION,
91+
group = "bufbuild",
92+
name = "protobuf-conformance",
93+
version = libs.versions.protobuf.asProvider().get().substringAfter("."),
94+
classifier = null,
95+
ext = null,
96+
configuration = null,
97+
) {
98+
artifact {
99+
name = "conformance_test_runner"
100+
type = "zip"
101+
extension = "zip"
102+
classifier = "$osPart-$archPart"
103+
}
104+
}
105+
}
106+
107+
val unzipProtobufConformance = tasks.register<Copy>(UNZIP_PROTOBUF_CONFORMANCE_TASK) {
108+
from(configurations.getByName(CONFORMANCE_TEST_RUNNER_CONFIGURATION).map {
109+
zipTree(it).matching { include("include/**") }
110+
})
111+
112+
val destDir = project.layout.projectDirectory
113+
.dir("src")
114+
.dir("test")
115+
.dir("proto")
116+
117+
into(destDir)
118+
119+
eachFile {
120+
if (path.startsWith("include/")) {
121+
path = "protobuf-conformance/${path.removePrefix("include/")}"
122+
}
123+
}
124+
125+
includeEmptyDirs = false
126+
127+
doFirst {
128+
destDir.asFile.deleteRecursively()
129+
}
130+
}
131+
132+
val writeConformanceExecutablePath =
133+
tasks.register<ConformanceExecutablePathWriter>(WRITE_CONFORMANCE_EXECUTABLE_PATH_TASK) {
134+
executable.set(
135+
configurations.getByName(CONFORMANCE_TEST_RUNNER_CONFIGURATION).map {
136+
zipTree(it).matching { include("bin/**") }.files.first()
137+
}.single()
138+
)
139+
140+
destination.set(
141+
project.layout.buildDirectory.file("generated/protobuf-conformance/executable-path.kt").get().asFile
142+
)
143+
}
144+
145+
tasks.matching { it.name == "processTestProtoFiles" }.all {
146+
dependsOn(unzipProtobufConformance)
147+
}
148+
149+
the<KotlinJvmExtension>().apply {
150+
sourceSets.getByName("test") {
151+
kotlin.srcDir(writeConformanceExecutablePath.map { it.destination.get().parentFile })
152+
}
153+
}
154+
}

‎gradle-plugin/build.gradle.kts

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,6 @@ gradlePlugin {
8383

8484
abstract class GeneratePluginVersionsTask @Inject constructor(
8585
@get:Input val libraryVersion: String,
86-
@get:Input val protobufVersion: String,
87-
@get:Input val grpcVersion: String,
88-
@get:Input val grpcKotlinVersion: String,
8986
@get:Input val bufToolVersion: String,
9087
@get:OutputDirectory val sourcesDir: File
9188
) : DefaultTask() {
@@ -104,24 +101,6 @@ package kotlinx.rpc
104101
*/
105102
public const val LIBRARY_VERSION: String = "$libraryVersion"
106103
107-
@Deprecated("Use kotlinx.rpc.LIBRARY_VERSION instead", ReplaceWith("kotlinx.rpc.LIBRARY_VERSION"))
108-
public const val PLUGIN_VERSION: String = LIBRARY_VERSION
109-
110-
/**
111-
* The version of the protobuf library.
112-
*/
113-
public const val PROTOBUF_VERSION: String = "$protobufVersion"
114-
115-
/**
116-
* The version of the grpc java library.
117-
*/
118-
public const val GRPC_VERSION: String = "$grpcVersion"
119-
120-
/**
121-
* The version of the grpc kotlin library.
122-
*/
123-
public const val GRPC_KOTLIN_VERSION: String = "$grpcKotlinVersion"
124-
125104
/**
126105
* The version of the buf tool used to generate protobuf.
127106
*/
@@ -141,18 +120,12 @@ val sourcesDir = File(project.layout.buildDirectory.asFile.get(), "generated-sou
141120
val generatePluginVersionsTask = tasks.register<GeneratePluginVersionsTask>(
142121
GeneratePluginVersionsTask.NAME,
143122
version.toString(),
144-
libs.versions.protobuf.asProvider().get(),
145-
libs.versions.grpc.asProvider().get(),
146-
libs.versions.grpc.kotlin.get(),
147123
libs.versions.buf.tool.get(),
148124
sourcesDir,
149125
)
150126

151127
abstract class GenerateTestVersionTask @Inject constructor(
152128
@get:Input val kotlinVersion: String,
153-
@get:Input val protobufVersion: String,
154-
@get:Input val grpcVersion: String,
155-
@get:Input val grpcKotlinVersion: String,
156129
@get:Input val buildRepo: String,
157130
@get:OutputDirectory val sourcesDir: File
158131
) : DefaultTask() {
@@ -170,11 +143,6 @@ const val KOTLIN_VERSION: String = "$kotlinVersion"
170143
171144
const val BUILD_REPO: String = "$buildRepo"
172145
173-
// can't use from generatePluginVersionsTask bacause Gradle messes up caches
174-
const val TEST_PROTOBUF_VERSION: String = "$protobufVersion"
175-
const val TEST_GRPC_VERSION: String = "$grpcVersion"
176-
const val TEST_GRPC_KOTLIN_VERSION: String = "$grpcKotlinVersion"
177-
178146
""".trimIndent()
179147
)
180148
}
@@ -191,9 +159,6 @@ val globalRootDir: String by extra
191159
val generateTestVersionsTask = tasks.register<GenerateTestVersionTask>(
192160
GenerateTestVersionTask.NAME,
193161
libs.versions.kotlin.lang.get(),
194-
libs.versions.protobuf.asProvider().get(),
195-
libs.versions.grpc.asProvider().get(),
196-
libs.versions.grpc.kotlin.get(),
197162
File(globalRootDir).resolve("build/repo").absolutePath,
198163
testSourcesDir,
199164
)

‎gradle-plugin/src/main/kotlin/kotlinx/rpc/Extensions.kt

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ import org.gradle.kotlin.dsl.property
1919
import java.util.concurrent.atomic.AtomicBoolean
2020
import javax.inject.Inject
2121

22-
internal fun Project.rpcExtension(): RpcExtension = extensions.findByType<RpcExtension>() ?: RpcExtension(objects, this)
22+
internal fun Project.rpcExtension(): RpcExtension = extensions.findByType<RpcExtension>()
23+
?: error("Rpc extension not found. Please apply the plugin to the project")
2324

2425
public open class RpcExtension @Inject constructor(objects: ObjectFactory, private val project: Project) {
2526
/**
@@ -51,6 +52,10 @@ public open class RpcExtension @Inject constructor(objects: ObjectFactory, priva
5152
* Protoc settings.
5253
*/
5354
public val protoc: ProtocExtension by lazy {
55+
if (protocApplied.get()) {
56+
error("Illegal access to protoc extension during DefaultProtocExtension.init")
57+
}
58+
5459
protocApplied.set(true)
5560
objects.newInstance<DefaultProtocExtension>()
5661
}

‎gradle-plugin/src/main/kotlin/kotlinx/rpc/buf/tasks/BufExecTask.kt

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -117,11 +117,11 @@ internal fun <T : BufExecTask> Project.registerBufExecTask(
117117
bufExecutable.set(executableConfiguration.singleFile)
118118
this.workingDir.set(workingDir)
119119

120-
val buf = project.rpcExtension().protoc.buf
121-
configFile.set(buf.configFile)
122-
logFormat.set(buf.logFormat)
123-
bufTimeoutInWholeSeconds.set(buf.timeout.map { it.inWholeSeconds })
124-
debug.set(project.gradle.startParameter.logLevel == LogLevel.DEBUG)
120+
val buf = provider { rpcExtension().protoc.buf }
121+
configFile.set(buf.flatMap { it.configFile })
122+
logFormat.set(buf.flatMap { it.logFormat })
123+
bufTimeoutInWholeSeconds.set(buf.flatMap { it.timeout.map { duration -> duration.inWholeSeconds } })
124+
debug.set(gradle.startParameter.logLevel == LogLevel.DEBUG)
125125

126126
configuration()
127127
}

‎gradle-plugin/src/main/kotlin/kotlinx/rpc/buf/tasks/BufGenerateTask.kt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,11 @@ internal fun Project.registerBufGenerateTask(
129129
group = PROTO_GROUP
130130
description = "Generates code from .proto files using 'buf generate'"
131131

132-
val generate = project.rpcExtension().protoc.buf.generate
132+
val generate = provider { rpcExtension().protoc.buf.generate }
133133

134-
includeImports.set(generate.includeImports)
135-
includeWkt.set(generate.includeWkt)
136-
errorFormat.set(generate.errorFormat)
134+
includeImports.set(generate.flatMap { it.includeImports })
135+
includeWkt.set(generate.flatMap { it.includeWkt })
136+
errorFormat.set(generate.flatMap { it.errorFormat })
137137

138138
this.outputDirectory.set(outputDirectory)
139139

‎gradle-plugin/src/main/kotlin/kotlinx/rpc/buf/tasks/GenerateBufGenYaml.kt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import org.gradle.api.GradleException
1414
import org.gradle.api.Project
1515
import org.gradle.api.provider.ListProperty
1616
import org.gradle.api.provider.Property
17+
import org.gradle.api.provider.Provider
1718
import org.gradle.api.tasks.Input
1819
import org.gradle.api.tasks.OutputFile
1920
import org.gradle.api.tasks.TaskAction
@@ -136,13 +137,13 @@ public abstract class GenerateBufGenYaml : DefaultTask() {
136137
internal fun Project.registerGenerateBufGenYamlTask(
137138
name: String,
138139
buildSourceSetsDir: File,
139-
protocPlugins: Iterable<ProtocPlugin>,
140+
protocPlugins: Provider<List<ProtocPlugin>>,
140141
configure: GenerateBufGenYaml.() -> Unit = {},
141142
): TaskProvider<GenerateBufGenYaml> {
142143
val capitalizeName = name.replaceFirstChar { it.uppercase() }
143144
return project.tasks.register<GenerateBufGenYaml>("${GenerateBufGenYaml.NAME_PREFIX}$capitalizeName") {
144145
val pluginsProvider = project.provider {
145-
protocPlugins.map { plugin ->
146+
protocPlugins.get().map { plugin ->
146147
if (!plugin.artifact.isPresent) {
147148
throw GradleException(
148149
"Artifact is not specified for protoc plugin ${plugin.name}. " +

‎gradle-plugin/src/main/kotlin/kotlinx/rpc/internal/configureLocalProtocGenDevelopmentDependency.kt

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,20 @@
44

55
package kotlinx.rpc.internal
66

7-
import kotlinx.rpc.RpcExtension
87
import kotlinx.rpc.buf.tasks.BufGenerateTask
98
import kotlinx.rpc.protoc.grpcKotlinMultiplatform
109
import kotlinx.rpc.protoc.kotlinMultiplatform
10+
import kotlinx.rpc.rpcExtension
1111
import org.gradle.api.Project
1212
import org.gradle.internal.extensions.core.extra
1313
import org.gradle.kotlin.dsl.provideDelegate
14-
import org.gradle.kotlin.dsl.the
1514
import org.gradle.kotlin.dsl.withType
1615

1716
@InternalRpcApi
1817
public fun Project.configureLocalProtocGenDevelopmentDependency() {
1918
val globalRootDir: String by extra
2019

21-
the<RpcExtension>().protoc.plugins {
20+
rpcExtension().protoc.plugins {
2221
kotlinMultiplatform {
2322
local {
2423
javaJar("$globalRootDir/protoc-gen/protobuf/build/libs/protobuf-$version-all.jar")

0 commit comments

Comments
 (0)