Skip to content

Commit 97c3cb1

Browse files
orangyorangy
committed
Run parameterized benchmarks on all platforms, support default values
1 parent 6fd0042 commit 97c3cb1

File tree

12 files changed

+283
-120
lines changed

12 files changed

+283
-120
lines changed

‎examples/kotlin-multiplatform/build.gradle

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,19 +99,22 @@ benchmark {
9999
configurations {
100100
main { // --> jvmBenchmark/jsBenchmark + benchmark
101101
iterations = 5 // number of iterations
102-
iterationTime = 300
102+
iterationTime = 300
103103
iterationTimeUnit = "ms"
104104
}
105105

106-
big { // --> jvmBigBenchmark + bigBenchmark(jvm+js+native)
107-
param("collectionSize", "1000000")
106+
params {
107+
iterations = 5 // number of iterations
108+
iterationTime = 300
109+
iterationTimeUnit = "ms"
110+
include("ParamBenchmark")
111+
param("data", 5, 1, 8)
112+
param("unused", 6, 9)
108113
}
109114

110115
fast { // --> jvmFastBenchmark
111116
include("Common")
112117
exclude("long")
113-
param("collectionSize", 10)
114-
param("repeats", 1000)
115118
iterations = 1
116119
iterationTime = 300 // time in ms per iteration
117120
iterationTimeUnit = "ms" // time in ms per iteration

‎examples/kotlin-multiplatform/src/commonMain/kotlin/ParamBenchmark.kt

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package test
2+
3+
import org.jetbrains.gradle.benchmarks.*
4+
import kotlin.math.*
5+
6+
@State(Scope.Benchmark)
7+
@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
8+
@OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS)
9+
@BenchmarkMode(Mode.Throughput)
10+
class ParamBenchmark {
11+
12+
@Param("1", "2")
13+
var data = 0
14+
15+
@Param("1", "2")
16+
var value = 0
17+
18+
private lateinit var text : String
19+
20+
@Setup
21+
fun setUp() {
22+
text = "Hello!"
23+
}
24+
25+
@Benchmark
26+
fun mathBenchmark(): Double {
27+
return log(sqrt(data.toDouble()) * data, 2.0)
28+
}
29+
30+
@Benchmark
31+
fun otherBenchmark(): Int {
32+
return data + data
33+
}
34+
}

‎plugin/main/src/org/jetbrains/gradle/benchmarks/SuiteSourceGenerator.kt

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,15 @@ package org.jetbrains.gradle.benchmarks
22

33
import com.squareup.kotlinpoet.*
44
import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy
5+
import org.jetbrains.kotlin.builtins.*
56
import org.jetbrains.kotlin.descriptors.*
67
import org.jetbrains.kotlin.name.*
78
import org.jetbrains.kotlin.resolve.*
89
import org.jetbrains.kotlin.resolve.annotations.*
910
import org.jetbrains.kotlin.resolve.constants.*
1011
import org.jetbrains.kotlin.resolve.descriptorUtil.*
1112
import org.jetbrains.kotlin.resolve.scopes.*
13+
import org.jetbrains.kotlin.types.*
1214
import java.io.*
1315

1416
enum class Platform {
@@ -19,6 +21,7 @@ class SuiteSourceGenerator(val title: String, val module: ModuleDescriptor, val
1921
companion object {
2022
val setupFunctionName = "setUp"
2123
val teardownFunctionName = "tearDown"
24+
val parametersFunctionName = "parametrize"
2225

2326
val externalConfigurationFQN = "org.jetbrains.gradle.benchmarks.ExternalConfiguration"
2427
val benchmarkAnnotationFQN = "org.jetbrains.gradle.benchmarks.Benchmark"
@@ -32,6 +35,7 @@ class SuiteSourceGenerator(val title: String, val module: ModuleDescriptor, val
3235
val outputTimeAnnotationFQN = "org.jetbrains.gradle.benchmarks.OutputTimeUnit"
3336
val warmupAnnotationFQN = "org.jetbrains.gradle.benchmarks.Warmup"
3437
val measureAnnotationFQN = "org.jetbrains.gradle.benchmarks.Measurement"
38+
val paramAnnotationFQN = "org.jetbrains.gradle.benchmarks.Param"
3539

3640
val mainBenchmarkPackage = "org.jetbrains.gradle.benchmarks.generated"
3741

@@ -104,6 +108,10 @@ class SuiteSourceGenerator(val title: String, val module: ModuleDescriptor, val
104108
val functions = DescriptorUtils.getAllDescriptors(original.unsubstitutedMemberScope)
105109
.filterIsInstance<FunctionDescriptor>()
106110

111+
val parameterProperties = DescriptorUtils.getAllDescriptors(original.unsubstitutedMemberScope)
112+
.filterIsInstance<PropertyDescriptor>()
113+
.filter { it.annotations.any { it.fqName.toString() == paramAnnotationFQN } }
114+
107115
val measureAnnotation = original.annotations.singleOrNull { it.fqName.toString() == measureAnnotationFQN }
108116
val warmupAnnotation = original.annotations.singleOrNull { it.fqName.toString() == warmupAnnotationFQN }
109117
val outputTimeAnnotation = original.annotations.singleOrNull { it.fqName.toString() == outputTimeAnnotationFQN }
@@ -157,21 +165,58 @@ class SuiteSourceGenerator(val title: String, val module: ModuleDescriptor, val
157165
}
158166
}
159167

168+
/*
169+
private fun parametrize(instance: ParamBenchmark, params: Map<String, String>) {
170+
instance.data = (params["data"] ?: error("No parameter value provided for property 'data'")).toInt()
171+
}
172+
*/
173+
function(parametersFunctionName) {
174+
addModifiers(KModifier.PRIVATE)
175+
addParameter("instance", originalClass)
176+
addParameter("params", MAP.parameterizedBy(STRING, STRING))
177+
parameterProperties.forEach { property ->
178+
val type = property.type.nameIfStandardType ?: error("Only simple types are supported and `${property.type}` is not.")
179+
addStatement("instance.${property.name} = params.getValue(\"${property.name}\").to$type()")
180+
}
181+
}
182+
183+
val defaultParameters = parameterProperties.associateBy({ it.name }, {
184+
val annotation = it.annotations.findAnnotation(FqName(paramAnnotationFQN))!!
185+
val constant = annotation.argumentValue("value")
186+
?: error("@Param annotation should have at least one default value")
187+
val values = constant.value as? List<String>
188+
?: error("@Param annotation should have at least one default value")
189+
values
190+
})
191+
192+
val defaultParametersString = defaultParameters.entries
193+
.joinToString(prefix = "mapOf(", postfix = ")") {
194+
"\"${it.key}\" to " + it.value.joinToString(prefix = "listOf(", postfix = ")")
195+
196+
}
197+
160198
val timeUnitClass = ClassName.bestGuess(timeUnitFQN)
161199
val iterationTimeClass = ClassName.bestGuess(iterationTimeFQN)
162200
val modeClass = ClassName.bestGuess(modeFQN)
163201

164202
function("describe") {
165203
returns(suiteDescriptorType.parameterizedBy(originalClass))
166204
addCode(
167-
"«val descriptor = %T(name = %S, factory = ::%T, setup = ::%N, teardown = ::%N",
205+
"«val descriptor = %T(name = %S, factory = ::%T, setup = ::%N, teardown = ::%N, parametrize = ::%N",
168206
suiteDescriptorType,
169207
originalName,
170208
originalClass,
171209
setupFunctionName,
172-
teardownFunctionName
210+
teardownFunctionName,
211+
parametersFunctionName
173212
)
174213

214+
val params =
215+
parameterProperties.joinToString(prefix = "listOf(", postfix = ")") { "\"${it.name}\"" }
216+
addCode(", parameters = $params")
217+
218+
addCode(", defaultParameters = $defaultParametersString")
219+
175220
if (iterations != null)
176221
addCode(", iterations = $iterations")
177222
if (warmups != null)
@@ -265,3 +310,6 @@ inline fun FileSpec.Builder.function(
265310
addFunction(it)
266311
}
267312
}
313+
314+
val KotlinType.nameIfStandardType: Name?
315+
get() = constructor.declarationDescriptor?.name

‎runtime/commonMain/src/org/jetbrains/gradle/benchmarks/CommonBenchmarkAnnotations.kt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,4 +79,6 @@ expect annotation class Measurement(
7979
val time: Int = -1,
8080
val timeUnit: BenchmarkTimeUnit = BenchmarkTimeUnit.SECONDS,
8181
val batchSize: Int = -1
82-
)
82+
)
83+
84+
expect annotation class Param(vararg val value: String)

‎runtime/commonMain/src/org/jetbrains/gradle/benchmarks/SuiteDescriptor.kt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,13 @@ package org.jetbrains.gradle.benchmarks
33
open class SuiteDescriptor<T>(
44
val name: String,
55
val factory: () -> T,
6+
val parametrize: (T, Map<String, String>) -> Unit,
67
val setup: (T) -> Unit,
78
val teardown: (T) -> Unit,
89

10+
val parameters: List<String>,
11+
val defaultParameters: Map<String, List<String>>,
12+
913
val iterations: Int = 3,
1014
val warmups: Int = 3,
1115

‎runtime/commonMain/src/org/jetbrains/gradle/benchmarks/SuiteExecutor.kt

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,5 +49,54 @@ abstract class SuiteExecutor(val executionName: String, arguments: Array<out Str
4949
benchmarks: List<BenchmarkDescriptor<Any?>>,
5050
complete: () -> Unit
5151
)
52+
53+
protected fun id(name: String, params: Map<String, String>): String {
54+
val id = if (params.isEmpty())
55+
name
56+
else
57+
name + params.entries.joinToString(prefix = " | ") { "${it.key}=${it.value}" }
58+
return id
59+
}
60+
}
61+
62+
fun runWithParameters(
63+
names: List<String>,
64+
parameters: Map<String, List<String>>,
65+
defaults: Map<String, List<String>>,
66+
function: (Map<String, String>) -> Unit
67+
) {
68+
if (names.isEmpty()) {
69+
function(mapOf())
70+
return
71+
}
72+
73+
fun parameterValues(name: String): List<String> {
74+
return parameters.getOrElse(name) {
75+
defaults.getOrElse(name) {
76+
error("No value specified for parameter '$name'")
77+
}
78+
}
79+
}
80+
81+
val valueIndices = IntArray(names.size)
82+
val valueLimits = IntArray(names.size) {
83+
val name = names[it]
84+
parameterValues(name).size
85+
}
86+
while (true) {
87+
val paramsVariant = names.indices.associateBy({ names[it] }, {
88+
parameterValues(names[it])[valueIndices[it]]
89+
})
90+
function(paramsVariant)
91+
for (index in valueIndices.indices) {
92+
valueIndices[index]++
93+
if (valueIndices[index] < valueLimits[index])
94+
break
95+
else
96+
if (index == valueIndices.lastIndex)
97+
return
98+
valueIndices[index] = 0
99+
}
100+
}
52101
}
53102

‎runtime/jsMain/src/org/jetbrains/gradle/benchmarks/JsBenchmarkAnnotations.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,5 @@ actual annotation class Measurement(
3636
actual val timeUnit: BenchmarkTimeUnit,
3737
actual val batchSize: Int
3838
)
39+
40+
actual annotation class Param(actual vararg val value: String)

0 commit comments

Comments
 (0)