Support registering multiplatform source sets as benchmark targets (#112)

Author: kyay10

README.md

@@ -99,7 +99,11 @@ kotlin {
 # Configuration
 
 In a `build.gradle` file create `benchmark` section, and inside it add a `targets` section.
-In this section register all targets you want to run benchmarks from. 
+In this section register all compilations you want to run benchmarks for. 
+`register` should either be called on the name of a target (e.g. `"jvm"`) which will register its `main` compilation
+(meaning that `register("jvm")` and `register("jvmMain")` register the same compilation)
+Or on the name of a source set (e.g. `"jvmTest"`, `"jsBenchmark"`) which will register the apt compilation
+(e.g. `register("jsFoo")` uses the `foo` compilation defined for the `js` target)
 Example for multiplatform project:
 
 ```groovy
@@ -211,6 +215,7 @@ benchmark {
 # Separate source sets for benchmarks
 
 Often you want to have benchmarks in the same project, but separated from main code, much like tests. Here is how:
+For a Kotlin/JVM project:
 
 Define source set:
 ```groovy
@@ -241,6 +246,27 @@ benchmark {
 }
 ```
 
+For a Kotlin Multiplatform project:
+
+Define a new compilation in whichever target you'd like (e.g. `jvm`, `js`, etc):
+```groovy
+kotlin {
+    jvm {
+        compilations.create('benchmark') { associateWith(compilations.main) }
+    }
+}
+```
+
+Register it by its source set name (`jvmBenchmark` is the name for the `benchmark` compilation for `jvm` target):
+
+```groovy
+benchmark {
+    targets {
+        register("jvmBenchmark")    
+    }
+}
+```
+
 # Examples
 
 The project contains [examples](https://github.com/Kotlin/kotlinx-benchmark/tree/master/examples) subproject that demonstrates using the library.

examples/kotlin-multiplatform/build.gradle

@@ -14,7 +14,9 @@ allOpen {
 }
 
 kotlin {
-    jvm()
+    jvm {
+        compilations.create('benchmark') { associateWith(compilations.main) }
+    }
     js('jsIr', IR) { nodejs() }
     js('jsIrBuiltIn', IR) { nodejs() }
     wasm { d8() }
@@ -101,10 +103,16 @@ benchmark {
 
     // Setup configurations
     targets {
-        // This one matches compilation base name, e.g. 'jvm', 'jvmTest', etc
+        // This one matches target name, e.g. 'jvm', 'js',
+        // and registers its 'main' compilation, so 'jvm' registers 'jvmMain'
         register("jvm") {
             jmhVersion = "1.21"
         }
+        // This one matches source set name, e.g. 'jvmMain', 'jvmTest', etc
+        // and register the corresponding compilation (here the 'benchmark' compilation declared in the 'jvm' target)
+        register("jvmBenchmark") {
+            jmhVersion = "1.21"
+        }
         register("jsIr")
         register("jsIrBuiltIn") {
             jsBenchmarksExecutor = JsBenchmarksExecutor.BuiltIn

examples/kotlin-multiplatform/src/jvmBenchmark/kotlin/JvmBenchmark.kt

@@ -0,0 +1,31 @@
+package test
+
+import org.openjdk.jmh.annotations.*
+import java.util.concurrent.*
+
+@State(Scope.Benchmark)
+@Fork(1)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+// jvmBenchmark can access declarations from jvmMain!
+@Warmup(iterations = WARMUP_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
+class JvmBenchmark {
+    private var data = 0.0
+
+    @Setup
+    fun setUp() {
+        data = 3.0
+    }
+
+    @Benchmark
+    fun sqrtBenchmark(): Double {
+        return Math.sqrt(data)
+    }
+
+    @Benchmark
+    fun cosBenchmark(): Double {
+        return Math.cos(data)
+    }
+}
+

examples/kotlin-multiplatform/src/jvmMain/kotlin/JvmTestBenchmark.kt

@@ -3,11 +3,12 @@ package test
 import org.openjdk.jmh.annotations.*
 import java.util.concurrent.*
 
+const val WARMUP_ITERATIONS = 20
 @State(Scope.Benchmark)
 @Fork(1)
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.NANOSECONDS)
-@Warmup(iterations = 20, time = 1, timeUnit = TimeUnit.SECONDS)
+@Warmup(iterations = WARMUP_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS)
 @Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
 class JvmTestBenchmark {
     private var data = 0.0

integration/src/main/kotlin/kotlinx/benchmark/integration/BenchmarkTarget.kt

@@ -0,0 +1,18 @@
+package kotlinx.benchmark.integration
+
+enum class JsBenchmarksExecutor {
+    BenchmarkJs,
+    BuiltIn
+}
+
+class BenchmarkTarget {
+    var jmhVersion: String? = null
+    var jsBenchmarksExecutor: JsBenchmarksExecutor? = null
+
+    fun lines(name: String): List<String> = """
+        register("$name") {
+            ${jmhVersion?.let { "jmhVersion = \"$it\"" } ?: ""}
+            ${jsBenchmarksExecutor?.let { "jsBenchmarksExecutor = JsBenchmarksExecutor.${it.name}" } ?: ""}
+        }
+    """.trimIndent().split("\n")
+}

integration/src/main/kotlin/kotlinx/benchmark/integration/ProjectBuilder.kt

@@ -4,10 +4,14 @@ import java.io.*
 
 class ProjectBuilder {
     private val configurations = mutableMapOf<String, BenchmarkConfiguration>()
+    private val targets = mutableMapOf<String, BenchmarkTarget>()
 
     fun configuration(name: String, configuration: BenchmarkConfiguration.() -> Unit = {}) {
         configurations[name] = BenchmarkConfiguration().apply(configuration)
     }
+    fun register(name: String, configuration: BenchmarkTarget.() -> Unit = {}) {
+        targets[name] = BenchmarkTarget().apply(configuration)
+    }
 
     fun build(original: String): String {
 
@@ -17,6 +21,9 @@ benchmark {
     configurations {
         ${configurations.flatMap { it.value.lines(it.key) }.joinToString("\n        ")}
     }
+    targets {
+        ${targets.flatMap { it.value.lines(it.key) }.joinToString("\n        ")}
+    }
 }
             """.trimIndent()
 

integration/src/test/kotlin/kotlinx/benchmark/integration/SourceSetAsBenchmarkTargetTest.kt

@@ -0,0 +1,29 @@
+package kotlinx.benchmark.integration
+
+import java.io.*
+import kotlin.test.*
+
+class SourceSetAsBenchmarkTargetTest : GradleTest() {
+
+    @Test
+    fun testSupportForSourceSetsAsBenchmarkTargets() {
+        val jvmBenchmark = "jvmBenchmark"
+        val configuration = "jsonDefault"
+        val targets = listOf("jsIr", "wasm", "jvm", "native", jvmBenchmark)
+
+        val runner =
+            project("kotlin-multiplatform", true) {
+                configuration(configuration) {
+                    iterations = 1
+                    iterationTime = 100
+                    iterationTimeUnit = "ms"
+                }
+                register(jvmBenchmark) { jmhVersion = "1.21" }
+            }
+
+        runner.run("${configuration}Benchmark")
+        val reports = reports(configuration)
+        assertEquals(targets.size, reports.size)
+        assertEquals(targets.map { "$it.json" }.toSet(), reports.map(File::getName).toSet())
+    }
+}

integration/src/test/resources/templates/kotlin-multiplatform/build.gradle

@@ -2,7 +2,9 @@ import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.HostManager
 
 kotlin {
-    jvm()
+    jvm {
+        compilations.create('benchmark') { associateWith(compilations.main) }
+    }
     js('jsIr', IR) { nodejs() }
     wasm { d8() }
 
@@ -22,6 +24,12 @@ kotlin {
                 implementation(benchmarkRuntimeJvm)
             }
         }
+        jvmBenchmark {
+            dependsOn(jvmMain)
+            dependencies {
+                implementation(benchmarkRuntimeJvm)
+            }
+        }
         jsIrMain {
             dependencies {
                 implementation(benchmarkRuntimeJsIr)

plugin/main/src/kotlinx/benchmark/gradle/BenchmarksExtension.kt

@@ -51,7 +51,9 @@ open class BenchmarksExtension(val project: Project) {
             when {
                 multiplatform != null -> {
                     val target = multiplatform.targets.findByName(name)
-                    when (val compilation = target?.compilations?.findByName(KotlinCompilation.MAIN_COMPILATION_NAME)) {
+                    // We allow the name to be either a target or a source set
+                    when (val compilation = target?.compilations?.findByName(KotlinCompilation.MAIN_COMPILATION_NAME)
+                        ?: multiplatform.targets.flatMap { it.compilations }.find { it.defaultSourceSetName == name }) {
                         null -> {
                             project.logger.warn("Warning: Cannot find a benchmark compilation '$name', ignoring.")
                             BenchmarkTarget(this, name) // ignore