Initial work on params

orangy · orangy · commit 6fd00423a1a2 · 2019-06-08T01:09:14.000+03:00
diff --git a/examples/java/build.gradle b/examples/java/build.gradle
@@ -8,6 +8,18 @@ dependencies {
 }
 
 benchmark {
+    configurations {
+        main { 
+            iterationTime = 300
+            iterationTimeUnit = "ms"
+        }
+        singleParam {
+            iterationTime = 300
+            iterationTimeUnit = "ms"
+            param("stringValue", "C", "D")
+            param("intValue", 1, 10)
+        }
+    }
     targets {
         register("main") {
             jmhVersion = "1.21"
diff --git a/examples/java/src/main/java/test/SampleJavaBenchmark.java b/examples/java/src/main/java/test/SampleJavaBenchmark.java
@@ -1,15 +1,22 @@
 package test;
 
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.*;
 
 @State(Scope.Benchmark)
+@Fork(1)
 public class SampleJavaBenchmark {
+    @Param({"A", "B"})
+    String stringValue;
+
+    @Param({"1", "2"})
+    int intValue;
+    
     @Benchmark
     public String stringBuilder() {
         StringBuilder stringBuilder = new StringBuilder();
         stringBuilder.append(10);
+        stringBuilder.append(stringValue);
+        stringBuilder.append(intValue);
         return stringBuilder.toString();
     }
 }
diff --git a/plugin/main/src/org/jetbrains/gradle/benchmarks/BenchmarkConfiguration.kt b/plugin/main/src/org/jetbrains/gradle/benchmarks/BenchmarkConfiguration.kt
@@ -5,14 +5,15 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.*
 
 open class BenchmarkConfiguration(val extension: BenchmarksExtension, val name: String) {
     var iterations: Int? = null
+    var warmups: Int? = null
     var iterationTime: Long? = null
     var iterationTimeUnit: String? = null
     var mode: String? = null
     var outputTimeUnit: String? = null
-    
+
     var includes: MutableList<String> = mutableListOf()
     var excludes: MutableList<String> = mutableListOf()
-    var params: MutableMap<String, Any?> = mutableMapOf()
+    var params: MutableMap<String, MutableList<Any?>> = mutableMapOf()
 
     fun include(pattern: String) {
         includes.add(pattern)
@@ -22,8 +23,9 @@ open class BenchmarkConfiguration(val extension: BenchmarksExtension, val name:
         excludes.add(pattern)
     }
 
-    fun param(name: String, value: Any?) {
-        params[name] = value
+    fun param(name: String, vararg value: Any?) {
+        val values = params.getOrPut(name) { mutableListOf() }
+        values.addAll(value)
     }
 
     fun capitalizedName() = if (name == "main") "" else name.capitalize()
@@ -34,11 +36,6 @@ open class BenchmarkTarget(val extension: BenchmarksExtension, val name: String)
     var workingDir: String? = null
 }
 
-class BenchmarkConfigurationDefaults {
-    var iterations = 10 // times
-    var iterationTime = 1L // seconds
-}
-
 abstract class JvmBenchmarkTarget(
     extension: BenchmarksExtension,
     name: String
diff --git a/plugin/main/src/org/jetbrains/gradle/benchmarks/JsNodeTasks.kt b/plugin/main/src/org/jetbrains/gradle/benchmarks/JsNodeTasks.kt
@@ -54,6 +54,7 @@ fun Project.createJsBenchmarkExecTask(
         arguments("-n", target.name)
         arguments("-r", reportFile.toString())
         config.iterations?.let { arguments("-i", it.toString()) }
+        config.warmups?.let { arguments("-w", it.toString()) }
         config.iterationTime?.let { arguments("-it", it.toString()) }
         config.iterationTimeUnit?.let { arguments("-itu", it) }
         config.outputTimeUnit?.let { arguments("-otu", it) }
@@ -65,8 +66,8 @@ fun Project.createJsBenchmarkExecTask(
         config.excludes.forEach {
             arguments("-E", it)
         }
-        config.params.forEach {
-            arguments("-P", "\"${it.key}=${it.value}\"")
+        config.params.forEach { (param, values) ->
+            values.forEach { value -> arguments("-P", "\"$param=$value\"") }
         }
 
         dependsOn("${target.name}${BenchmarksPlugin.BENCHMARK_DEPENDENCIES_SUFFIX}")
diff --git a/plugin/main/src/org/jetbrains/gradle/benchmarks/JvmTasks.kt b/plugin/main/src/org/jetbrains/gradle/benchmarks/JvmTasks.kt
@@ -111,6 +111,7 @@ fun Project.createJvmBenchmarkExecTask(
         args("-n", target.name)
         args("-r", reportFile.toString())
         config.iterations?.let { args("-i", it.toString()) }
+        config.warmups?.let { args("-w", it.toString()) }
         config.iterationTime?.let { args("-it", it.toString()) }
         config.iterationTimeUnit?.let { args("-itu", it) }
         config.outputTimeUnit?.let { args("-otu", it) }
@@ -122,8 +123,8 @@ fun Project.createJvmBenchmarkExecTask(
         config.excludes.forEach {
             args("-E", it)
         }
-        config.params.forEach {
-            args("-P", "\"${it.key}=${it.value}\"")
+        config.params.forEach { (param, values) ->
+            values.forEach { value -> args("-P", "\"$param=$value\"") }
         }
 
         dependsOn("${target.name}${BenchmarksPlugin.BENCHMARK_COMPILE_SUFFIX}")
diff --git a/plugin/main/src/org/jetbrains/gradle/benchmarks/NativeMultiplatformTasks.kt b/plugin/main/src/org/jetbrains/gradle/benchmarks/NativeMultiplatformTasks.kt
@@ -115,6 +115,7 @@ fun Project.createNativeBenchmarkExecTask(
         args("-n", target.name)
         args("-r", reportFile.toString())
         config.iterations?.let { args("-i", it.toString()) }
+        config.warmups?.let { args("-w", it.toString()) }
         config.iterationTime?.let { args("-it", it.toString()) }
         config.iterationTimeUnit?.let { args("-itu", it) }
         config.outputTimeUnit?.let { args("-otu", it) }
@@ -126,8 +127,8 @@ fun Project.createNativeBenchmarkExecTask(
         config.excludes.forEach {
             args("-E", it)
         }
-        config.params.forEach {
-            args("-P", "\"${it.key}=${it.value}\"")
+        config.params.forEach { (param, values) ->
+            values.forEach { value -> args("-P", "\"$param=$value\"") }
         }
 
         dependsOn(linkTask)
diff --git a/runtime/commonMain/src/org/jetbrains/gradle/benchmarks/RunnerConfiguration.kt b/runtime/commonMain/src/org/jetbrains/gradle/benchmarks/RunnerConfiguration.kt
@@ -11,7 +11,7 @@ class RunnerConfiguration : CommandLineInterface("Client") {
         "-P",
         "<name>=<value>",
         "Specifies parameter with the given name and value"
-    ).toKeyValuePair().storeToMap()
+    ).toKeyValuePair().storeToMultimap()
 
     val include by onFlagValue("-I", "<pattern>", "Include benchmarks matching the given pattern").storeToList()
     val exclude by onFlagValue("-E", "<pattern>", "Include benchmarks matching the given pattern").storeToList()
@@ -55,16 +55,16 @@ class RunnerConfiguration : CommandLineInterface("Client") {
 
     private fun Event<String>.toKeyValuePair() =
         map {
-            val parts = it.split('=', limit = 2)
+            val parts = it.removeSurrounding("\"").split('=', limit = 2)
             if (parts.size == 1)
                 parts[0] to ""
             else
                 parts[0] to parts[1]
         }
 
-    private fun Event<Pair<String, String>>.storeToMap(): ArgumentValue<Map<String, String>> {
-        val map = hashMapOf<String, String>()
-        add { (key, value) -> map[key] = value }
+    private fun Event<Pair<String, String>>.storeToMultimap(): ArgumentValue<Map<String, List<String>>> {
+        val map = hashMapOf<String, MutableList<String>>()
+        add { (key, value) -> map.getOrPut(key) { mutableListOf() }.add(value) }
         return ArgumentStorage(map)
     }
 
@@ -82,7 +82,7 @@ class RunnerConfiguration : CommandLineInterface("Client") {
 
     class LateInitArgumentStorage<T : Any>() : ArgumentValue<T> {
         private lateinit var value: T
-        
+
         override fun getValue(thisRef: Any?, prop: Any?): T =
             value
 
@@ -91,7 +91,6 @@ class RunnerConfiguration : CommandLineInterface("Client") {
         }
     }
 
-
     override fun toString(): String {
         return """$name -> $reportFile ($traceFormat)
 params: ${params.entries.joinToString(prefix = "{", postfix = "}") { "${it.key}: ${it.value}" }}
diff --git a/runtime/jvmMain/src/org/jetbrains/gradle/benchmarks/jvm/JvmBenchmarkRunner.kt b/runtime/jvmMain/src/org/jetbrains/gradle/benchmarks/jvm/JvmBenchmarkRunner.kt
@@ -27,14 +27,18 @@ fun main(args: Array<String>) {
     params.mode?.let {
         jmhOptions.mode(it)
     }
-    
-    params.include.forEach { 
+
+    params.include.forEach {
         jmhOptions.include(it)
     }
-    params.exclude.forEach { 
+    params.exclude.forEach {
         jmhOptions.exclude(it)
     }
 
+    params.params.forEach { (key, value) ->
+        jmhOptions.param(key, *value.toTypedArray())
+    }
+
     val reporter = BenchmarkProgress.create(params.traceFormat)
     val output = JmhOutputFormat(reporter, params.name)
     try {
@@ -55,9 +59,9 @@ fun main(args: Array<String>) {
 
 class JmhOutputFormat(private val reporter: BenchmarkProgress, private val suiteName: String) :
     PrintOutputFormat(System.out) {
-    
+
     internal var lastBenchmarkStart = ""
-    
+
     override fun startRun() {
         reporter.startSuite(suiteName)
     }
@@ -67,22 +71,32 @@ class JmhOutputFormat(private val reporter: BenchmarkProgress, private val suite
     }
 
     override fun startBenchmark(benchParams: BenchmarkParams) {
-        val benchmarkFQN = benchParams.benchmark
-        reporter.startBenchmark(suiteName, benchmarkFQN)
-        lastBenchmarkStart = benchmarkFQN
+        val benchmarkId = getBenchmarkId(benchParams)
+        reporter.startBenchmark(suiteName, benchmarkId)
+        lastBenchmarkStart = benchmarkId
     }
 
     override fun endBenchmark(result: BenchmarkResult?) {
         if (result != null) {
-            val benchmarkFQN = result.params.benchmark
+            val benchmarkId = getBenchmarkId(result.params)
             val value = result.primaryResult
             val message = value.extendedInfo().trim()
-            reporter.endBenchmark(suiteName, benchmarkFQN, BenchmarkProgress.FinishStatus.Success, message)
+            reporter.endBenchmark(suiteName, benchmarkId, BenchmarkProgress.FinishStatus.Success, message)
         } else {
             reporter.endBenchmarkException(suiteName, lastBenchmarkStart, "<ERROR>", "")
         }
     }
 
+    private fun getBenchmarkId(params: BenchmarkParams): String {
+        val benchmarkName = params.benchmark
+        val paramKeys = params.paramsKeys
+        val benchmarkId = if (paramKeys.isEmpty())
+            benchmarkName
+        else
+            benchmarkName + paramKeys.joinToString(prefix = " | ") { "$it=${params.getParam(it)}" }
+        return benchmarkId
+    }
+
     override fun iteration(benchParams: BenchmarkParams, params: IterationParams, iteration: Int) {}
 
     override fun iterationResult(
@@ -98,7 +112,7 @@ class JmhOutputFormat(private val reporter: BenchmarkProgress, private val suite
         }
         flush()
     }
-    
+
     override fun println(s: String) {
         if (!s.startsWith("#"))
             reporter.output(suiteName, lastBenchmarkStart, s)
