Skip to content

Commit 9d5b704

Browse files
Mr3zeeMr3zee
committed
Set up protobuf conformance tests (#447)
1 parent 91b2fde commit 9d5b704

File tree

72 files changed

+17418
-310
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+17418
-310
lines changed

‎.github/workflows/protobuf-conformance.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Verify Protobuf Conformance is Up-to-Date
2+
3+
on:
4+
pull_request:
5+
6+
permissions:
7+
contents: read
8+
9+
jobs:
10+
verify-platforms-table:
11+
name: Run Verification
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout Sources
15+
uses: actions/checkout@v4
16+
with:
17+
fetch-depth: 0
18+
- name: Setup Gradle
19+
uses: gradle/actions/setup-gradle@v4
20+
- name: Run Protobuf Conformance Test Generation
21+
run: ./gradlew tests:protobuf-conformance:bufGenerateTest --info --stacktrace
22+
- name: Check if protobuf-conformance test is up-to-date
23+
run: |
24+
if [[ -n "$(git status --porcelain | grep tests/protobuf-conformance/)" ]]; then
25+
echo "Protobuf conformance test is not up to date. Please run './gradlew tests:protobuf-conformance:bufGenerateTest' and commit changes"
26+
exit 1
27+
fi

‎.github/workflows/protobuf-well-known-types.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Verify Protobuf Well-Known Types are Up-to-Date
2+
3+
on:
4+
pull_request:
5+
6+
permissions:
7+
contents: read
8+
9+
jobs:
10+
verify-platforms-table:
11+
name: Run Verification
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout Sources
15+
uses: actions/checkout@v4
16+
with:
17+
fetch-depth: 0
18+
- name: Setup Gradle
19+
uses: gradle/actions/setup-gradle@v4
20+
- name: Run Protobuf Conformance Test Generation
21+
run: ./gradlew :protobuf:protobuf-core:bufGenerateCommonMain --info --stacktrace
22+
- name: Check if Well-Known Types are up-to-date
23+
run: |
24+
if [[ -n "$(git status --porcelain | grep protobuf/protobuf-core/)" ]]; then
25+
echo "Well-Known Types are not up-to-date. Please run './gradlew :protobuf:protobuf-core:bufGenerateCommonMain' and commit changes"
26+
exit 1
27+
fi

‎gradle-conventions/src/main/kotlin/util/tasks/protobufConformanceUpdate.kt‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
/*
2+
* Copyright 2023-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
package util.tasks
6+
7+
import org.gradle.api.DefaultTask
8+
import org.gradle.api.Project
9+
import org.gradle.api.provider.Property
10+
import org.gradle.api.tasks.Copy
11+
import org.gradle.api.tasks.Input
12+
import org.gradle.api.tasks.InputFile
13+
import org.gradle.api.tasks.OutputFile
14+
import org.gradle.api.tasks.TaskAction
15+
import org.gradle.kotlin.dsl.accessors.runtime.addExternalModuleDependencyTo
16+
import org.gradle.kotlin.dsl.dependencies
17+
import org.gradle.kotlin.dsl.register
18+
import org.gradle.kotlin.dsl.the
19+
import org.jetbrains.kotlin.gradle.dsl.KotlinJvmExtension
20+
import util.other.libs
21+
import java.io.File
22+
23+
const val CONFORMANCE_TEST_RUNNER_CONFIGURATION = "conformanceTestRunner"
24+
const val UNZIP_PROTOBUF_CONFORMANCE_TASK = "unzipProtobufConformance"
25+
const val WRITE_CONFORMANCE_EXECUTABLE_PATH_TASK = "writeConformanceExecutablePath"
26+
27+
abstract class ConformanceExecutablePathWriter : DefaultTask() {
28+
@get:Input
29+
abstract val outputDir: Property<File>
30+
31+
@get:Input
32+
abstract val kotlinTestDir: Property<File>
33+
34+
@get:InputFile
35+
abstract val executable: Property<File>
36+
37+
@get:OutputFile
38+
abstract val destination: Property<File>
39+
40+
@TaskAction
41+
fun action() {
42+
val dest = destination.get()
43+
if (dest.exists()) {
44+
dest.delete()
45+
}
46+
47+
val parent = dest.parentFile
48+
if (!parent.exists()) {
49+
parent.mkdirs()
50+
}
51+
52+
dest.writeText(
53+
"""
54+
// file generated by ConformanceExecutablePathWriter task
55+
56+
const val CONFORMANCE_EXECUTABLE_PATH: String = "${executable.get().absolutePath}"
57+
58+
const val CONFORMANCE_OUTPUT_DIR: String = "${outputDir.get().absolutePath}"
59+
60+
const val KOTLIN_TEST_DIR: String = "${kotlinTestDir.get().absolutePath}"
61+
62+
""".trimIndent()
63+
)
64+
}
65+
}
66+
67+
fun Project.setupProtobufConformanceResources() {
68+
val os = System.getProperty("os.name").lowercase()
69+
val osPart = when {
70+
os.startsWith("linux") -> "linux"
71+
os.startsWith("mac") -> "osx"
72+
else -> error("unsupported os for protobuf-conformance tests: $os")
73+
}
74+
75+
val archPart = when (val arch = System.getProperty("os.arch").lowercase()) {
76+
in setOf("x86_64", "amd64", "aarch64") -> "x86_64"
77+
else -> error("unsupported arch for protobuf-conformance tests: $arch")
78+
}
79+
80+
// https://stackoverflow.com/questions/23023069/gradle-download-and-unzip-file-from-url
81+
repositories.ivy {
82+
name = "protobuf-conformance-github"
83+
url = uri("https://github.com")
84+
85+
patternLayout {
86+
// https://github.com/bufbuild/protobuf-conformance/releases/download/v${version}/conformance_test_runner-${version}-${build}.zip
87+
artifact("[organisation]/[module]/releases/download/v[revision]/[artifact]-[revision]-[classifier].[ext]")
88+
}
89+
90+
// This is required in Gradle 6.0+ as metadata file (ivy.xml)
91+
// is mandatory. Docs linked below this code section
92+
metadataSources { artifact() }
93+
}
94+
95+
configurations.create(CONFORMANCE_TEST_RUNNER_CONFIGURATION)
96+
97+
// https://docs.gradle.org/current/javadoc/org/gradle/api/artifacts/dsl/DependencyHandler.html
98+
dependencies {
99+
addExternalModuleDependencyTo(
100+
this,
101+
CONFORMANCE_TEST_RUNNER_CONFIGURATION,
102+
group = "bufbuild",
103+
name = "protobuf-conformance",
104+
version = libs.versions.protobuf.asProvider().get().substringAfter("."),
105+
classifier = null,
106+
ext = null,
107+
configuration = null,
108+
) {
109+
artifact {
110+
name = "conformance_test_runner"
111+
type = "zip"
112+
extension = "zip"
113+
classifier = "$osPart-$archPart"
114+
}
115+
}
116+
}
117+
118+
val unzipProtobufConformance = tasks.register<Copy>(UNZIP_PROTOBUF_CONFORMANCE_TASK) {
119+
from(configurations.getByName(CONFORMANCE_TEST_RUNNER_CONFIGURATION).map {
120+
zipTree(it).matching { include("include/**") }
121+
})
122+
123+
val destDir = project.layout.projectDirectory
124+
.dir("src")
125+
.dir("main")
126+
.dir("proto")
127+
128+
into(destDir)
129+
130+
eachFile {
131+
if (path.startsWith("include/")) {
132+
path = "protobuf-conformance/${path.removePrefix("include/")}"
133+
}
134+
}
135+
136+
includeEmptyDirs = false
137+
138+
doFirst {
139+
destDir.asFile.deleteRecursively()
140+
}
141+
}
142+
143+
val writeConformanceExecutablePath =
144+
tasks.register<ConformanceExecutablePathWriter>(WRITE_CONFORMANCE_EXECUTABLE_PATH_TASK) {
145+
outputDir.set(
146+
project.layout.buildDirectory.get()
147+
.dir("protobuf-conformance")
148+
.asFile
149+
)
150+
151+
kotlinTestDir.set(
152+
project.layout.projectDirectory
153+
.dir("src")
154+
.dir("test")
155+
.dir("kotlin")
156+
.asFile
157+
)
158+
159+
executable.set(
160+
configurations.getByName(CONFORMANCE_TEST_RUNNER_CONFIGURATION).map {
161+
zipTree(it).matching { include("bin/**") }.files.first()
162+
}.single()
163+
)
164+
165+
destination.set(
166+
project.layout.buildDirectory.get()
167+
.dir("generated")
168+
.dir("protobuf-conformance")
169+
.file("executable-paths.kt")
170+
.asFile
171+
)
172+
}
173+
174+
tasks.matching { it.name == "processMainProtoFiles" || it.name == "processTestImportProtoFiles" }.all {
175+
dependsOn(unzipProtobufConformance)
176+
}
177+
178+
the<KotlinJvmExtension>().apply {
179+
sourceSets.getByName("main") {
180+
kotlin.srcDir(writeConformanceExecutablePath.map { it.destination.get().parentFile })
181+
}
182+
}
183+
}

‎gradle-plugin/api/gradle-plugin.api‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,7 @@ public class kotlinx/rpc/RpcStrictModeExtension {
3939

4040
public final class kotlinx/rpc/VersionsKt {
4141
public static final field BUF_TOOL_VERSION Ljava/lang/String;
42-
public static final field GRPC_KOTLIN_VERSION Ljava/lang/String;
43-
public static final field GRPC_VERSION Ljava/lang/String;
4442
public static final field LIBRARY_VERSION Ljava/lang/String;
45-
public static final field PLUGIN_VERSION Ljava/lang/String;
46-
public static final field PROTOBUF_VERSION Ljava/lang/String;
4743
}
4844

4945
public class kotlinx/rpc/buf/BufExtension {
@@ -179,17 +175,15 @@ public final class kotlinx/rpc/protoc/ProcessProtoFilesKt$inlined$sam$i$org_grad
179175

180176
public abstract interface class kotlinx/rpc/protoc/ProtoSourceSet {
181177
public abstract fun getName ()Ljava/lang/String;
178+
public abstract fun getPlugins ()Lorg/gradle/api/NamedDomainObjectContainer;
182179
public abstract fun getProto ()Lorg/gradle/api/file/SourceDirectorySet;
180+
public abstract fun plugins (Lorg/gradle/api/Action;)V
183181
public fun proto (Lorg/gradle/api/Action;)V
184-
public abstract fun protocPlugin (Lkotlinx/rpc/protoc/ProtocPlugin;)V
185-
public abstract fun protocPlugin (Lorg/gradle/api/NamedDomainObjectProvider;)V
186182
}
187183

188184
public abstract interface class kotlinx/rpc/protoc/ProtocExtension {
189185
public abstract fun buf (Lorg/gradle/api/Action;)V
190186
public abstract fun getBuf ()Lkotlinx/rpc/buf/BufExtension;
191-
public abstract fun getPlugins ()Lorg/gradle/api/NamedDomainObjectContainer;
192-
public abstract fun plugins (Lorg/gradle/api/Action;)V
193187
}
194188

195189
public class kotlinx/rpc/protoc/ProtocPlugin {

‎gradle-plugin/build.gradle.kts‎

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,6 @@ gradlePlugin {
8282

8383
abstract class GeneratePluginVersionsTask @Inject constructor(
8484
@get:Input val libraryVersion: String,
85-
@get:Input val protobufVersion: String,
86-
@get:Input val grpcVersion: String,
87-
@get:Input val grpcKotlinVersion: String,
8885
@get:Input val bufToolVersion: String,
8986
@get:OutputDirectory val sourcesDir: File
9087
) : DefaultTask() {
@@ -103,24 +100,6 @@ package kotlinx.rpc
103100
*/
104101
public const val LIBRARY_VERSION: String = "$libraryVersion"
105102
106-
@Deprecated("Use kotlinx.rpc.LIBRARY_VERSION instead", ReplaceWith("kotlinx.rpc.LIBRARY_VERSION"))
107-
public const val PLUGIN_VERSION: String = LIBRARY_VERSION
108-
109-
/**
110-
* The version of the protobuf library.
111-
*/
112-
public const val PROTOBUF_VERSION: String = "$protobufVersion"
113-
114-
/**
115-
* The version of the grpc java library.
116-
*/
117-
public const val GRPC_VERSION: String = "$grpcVersion"
118-
119-
/**
120-
* The version of the grpc kotlin library.
121-
*/
122-
public const val GRPC_KOTLIN_VERSION: String = "$grpcKotlinVersion"
123-
124103
/**
125104
* The version of the buf tool used to generate protobuf.
126105
*/
@@ -140,18 +119,12 @@ val sourcesDir = File(project.layout.buildDirectory.asFile.get(), "generated-sou
140119
val generatePluginVersionsTask = tasks.register<GeneratePluginVersionsTask>(
141120
GeneratePluginVersionsTask.NAME,
142121
version.toString(),
143-
libs.versions.protobuf.asProvider().get(),
144-
libs.versions.grpc.asProvider().get(),
145-
libs.versions.grpc.kotlin.get(),
146122
libs.versions.buf.tool.get(),
147123
sourcesDir,
148124
)
149125

150126
abstract class GenerateTestVersionTask @Inject constructor(
151127
@get:Input val kotlinVersion: String,
152-
@get:Input val protobufVersion: String,
153-
@get:Input val grpcVersion: String,
154-
@get:Input val grpcKotlinVersion: String,
155128
@get:Input val buildRepo: String,
156129
@get:OutputDirectory val sourcesDir: File
157130
) : DefaultTask() {
@@ -169,11 +142,6 @@ const val KOTLIN_VERSION: String = "$kotlinVersion"
169142
170143
const val BUILD_REPO: String = "$buildRepo"
171144
172-
// can't use from generatePluginVersionsTask bacause Gradle messes up caches
173-
const val TEST_PROTOBUF_VERSION: String = "$protobufVersion"
174-
const val TEST_GRPC_VERSION: String = "$grpcVersion"
175-
const val TEST_GRPC_KOTLIN_VERSION: String = "$grpcKotlinVersion"
176-
177145
""".trimIndent()
178146
)
179147
}
@@ -190,9 +158,6 @@ val globalRootDir: String by extra
190158
val generateTestVersionsTask = tasks.register<GenerateTestVersionTask>(
191159
GenerateTestVersionTask.NAME,
192160
libs.versions.kotlin.lang.get(),
193-
libs.versions.protobuf.asProvider().get(),
194-
libs.versions.grpc.asProvider().get(),
195-
libs.versions.grpc.kotlin.get(),
196161
File(globalRootDir).resolve("build/repo").absolutePath,
197162
testSourcesDir,
198163
)

‎gradle-plugin/src/main/kotlin/kotlinx/rpc/Extensions.kt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ import org.gradle.kotlin.dsl.property
1919
import java.util.concurrent.atomic.AtomicBoolean
2020
import javax.inject.Inject
2121

22-
internal fun Project.rpcExtension(): RpcExtension = extensions.findByType<RpcExtension>() ?: RpcExtension(objects, this)
22+
internal fun Project.rpcExtension(): RpcExtension = extensions.findByType<RpcExtension>()
23+
?: error("Rpc extension not found. Please apply the plugin to the project")
2324

2425
public open class RpcExtension @Inject constructor(objects: ObjectFactory, private val project: Project) {
2526
/**
@@ -51,6 +52,10 @@ public open class RpcExtension @Inject constructor(objects: ObjectFactory, priva
5152
* Protoc settings.
5253
*/
5354
public val protoc: ProtocExtension by lazy {
55+
if (protocApplied.get()) {
56+
error("Illegal access to protoc extension during DefaultProtocExtension.init")
57+
}
58+
5459
protocApplied.set(true)
5560
objects.newInstance<DefaultProtocExtension>()
5661
}

‎gradle-plugin/src/main/kotlin/kotlinx/rpc/buf/tasks/BufExecTask.kt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -117,11 +117,11 @@ internal fun <T : BufExecTask> Project.registerBufExecTask(
117117
bufExecutable.set(executableConfiguration.singleFile)
118118
this.workingDir.set(workingDir)
119119

120-
val buf = project.rpcExtension().protoc.buf
121-
configFile.set(buf.configFile)
122-
logFormat.set(buf.logFormat)
123-
bufTimeoutInWholeSeconds.set(buf.timeout.map { it.inWholeSeconds })
124-
debug.set(project.gradle.startParameter.logLevel == LogLevel.DEBUG)
120+
val buf = provider { rpcExtension().protoc.buf }
121+
configFile.set(buf.flatMap { it.configFile })
122+
logFormat.set(buf.flatMap { it.logFormat })
123+
bufTimeoutInWholeSeconds.set(buf.flatMap { it.timeout.map { duration -> duration.inWholeSeconds } })
124+
debug.set(gradle.startParameter.logLevel == LogLevel.DEBUG)
125125

126126
configuration()
127127
}

0 commit comments

Comments
 (0)