Fix samples, add main configuration by default, handle empty includes, configure mode & output time unit

orangy · orangy · commit 6b17c0c15514 · 2019-06-05T11:23:35.000+03:00
diff --git a/README.md b/README.md
@@ -57,18 +57,14 @@ repositories {
 }
 ```
 
-> Please note that in this early project stage only macOS x64 binaries are published. 
-If you need to run it on Linux or Windows, check out sources, update `build.gradle` for `runtime` project, and publish
-to Maven Local repository.   
+# Configuring benchmark targets
 
-# Configuring benchmark source sets
-
-In a `build.gradle` file create ` benchmark` section, and inside it add a `configurations` section.
+In a `build.gradle` file create ` benchmark` section, and inside it add a `targets` section.
 Example for multiplatform project:
 
 ```groovy
 benchmark {
-    configurations {
+    targets {
         register("jvm") 
         register("js")
         register("native")
@@ -80,7 +76,7 @@ Example for plain Java project:
 
 ```groovy
 benchmark {
-    configurations {
+    targets {
         register("main") 
     }
 }
@@ -90,26 +86,31 @@ Configure benchmarks:
 
 ```groovy
 benchmark {
-    defaults { // specify defaults for all configurations
-        iterations = 10 // number of iterations
-        iterationTime = 1000 // time in ms per iteration
+    // Create configurations
+    configurations {
+        main { // main configuration is created automatically, but you can change its defaults
+            iterations = 10 // number of iterations
+            iterationTime = 3 // time in seconds per iteration
+        }
+        fast {
+            warmups = 5 // number of warmup iterations
+            iterations = 3 // number of iterations
+            iterationTime = 500 // time in seconds per iteration
+            iterationTimeUnit = "ms" // time unity for iterationTime, default is seconds
+        }   
     }
     
-    // Setup configurations
-    configurations {
+    // Setup targets
+    targets {
         // This one matches compilation base name, e.g. 'jvm', 'jvmTest', etc
         register("jvm") {
             jmhVersion = "1.21" // available only for JVM compilations & Java source sets
-            
-            // for now, we use iterations and iterationTime for both warmup and measurements
         }
         register("js") {
             // Note, that benchmarks.js uses a different approach of minTime & maxTime and run benchmarks
             // until results are stable. We estimate minTime as iterationTime and maxTime as iterationTime*iterations
         }
-        register("native") {
-            iterationTime = 2000 // override the default
-        }
+        register("native")
     }
 }
 ```
@@ -129,7 +130,7 @@ Propagate dependencies and output from `main` sourceSet.
 
 ```groovy
 dependencies {
-    benchmarksCompile sourceSets.main.output + sourceSets.main.compileClasspath 
+    benchmarksCompile sourceSets.main.output + sourceSets.main.runtimeClasspath 
 }
 ```
 
@@ -141,7 +142,7 @@ Register `benchmarks` source set:
 
 ```groovy
 benchmarks {
-    configurations {
+    targets {
         register("benchmarks")    
     }
 }
diff --git a/examples/kotlin-multiplatform/build.gradle b/examples/kotlin-multiplatform/build.gradle
@@ -99,16 +99,17 @@ benchmark {
     configurations {
         main { // --> jvmBenchmark/jsBenchmark + benchmark
             iterations = 5 // number of iterations
-            iterationTime = 300 // time in ms per iteration
+            iterationTime = 300  
+            iterationTimeUnit = "ms"
         }
 
         big { // --> jvmBigBenchmark + bigBenchmark(jvm+js+native)
             param("collectionSize", "1000000")
         }
 
         fast { // --> jvmFastBenchmark
-            include(".*Common.*")
-            exclude(".*long.*")
+            include("Common")
+            exclude("long")
             param("collectionSize", 10)
             param("repeats", 1000)
             iterations = 1
diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle
@@ -21,7 +21,7 @@ sourceSets {
 // You can add "+ sourceSets.test.output" to include test output as well
 dependencies {
     implementation(project(":runtime"))
-    benchmarksCompile sourceSets.main.output + sourceSets.main.compileClasspath 
+    benchmarksCompile sourceSets.main.output + sourceSets.main.runtimeClasspath 
 }
 
 // Configure benchmark
diff --git a/plugin/main/src/org/jetbrains/gradle/benchmarks/BenchmarkConfiguration.kt b/plugin/main/src/org/jetbrains/gradle/benchmarks/BenchmarkConfiguration.kt
@@ -7,6 +7,9 @@ open class BenchmarkConfiguration(val extension: BenchmarksExtension, val name:
     var iterations: Int? = null
     var iterationTime: Long? = null
     var iterationTimeUnit: String? = null
+    var mode: String? = null
+    var outputTimeUnit: String? = null
+    
     var includes: MutableList<String> = mutableListOf()
     var excludes: MutableList<String> = mutableListOf()
     var params: MutableMap<String, Any?> = mutableMapOf()
@@ -23,23 +26,6 @@ open class BenchmarkConfiguration(val extension: BenchmarksExtension, val name:
         params[name] = value
     }
 
-    // TODO: this is error prone. User should use the mutable variables above. Plugin code should use these methods
-    // Basically, we want external properties, such as `-P` to override what's in build script
-    // We also want to use defaults if per configuration setting is missing 
-    fun iterations(): Int {
-        val externalSetting = extension.project.findProperty("benchmarks_iterations") as? String
-        if (externalSetting != null)
-            return externalSetting.toInt()
-        return iterations ?: extension.defaults.iterations
-    }
-
-    fun iterationTime(): Long {
-        val externalSetting = extension.project.findProperty("benchmarks_iterationTime") as? String
-        if (externalSetting != null)
-            return externalSetting.toLong()
-        return iterationTime ?: extension.defaults.iterationTime
-    }
-
     fun capitalizedName() = if (name == "main") "" else name.capitalize()
     fun prefixName(suffix: String) = if (name == "main") suffix else name + suffix.capitalize()
 }
diff --git a/plugin/main/src/org/jetbrains/gradle/benchmarks/BenchmarksExtension.kt b/plugin/main/src/org/jetbrains/gradle/benchmarks/BenchmarksExtension.kt
@@ -17,24 +17,16 @@ open class BenchmarksExtension(val project: Project) {
 
     val version = BenchmarksPlugin.PLUGIN_VERSION
 
-    val defaults = BenchmarkConfigurationDefaults()
-    
-    fun defaults(configure: Action<BenchmarkConfigurationDefaults>) {
-        configure.execute(defaults)
-    }
-    
-    fun defaults(configure: Closure<BenchmarkConfigurationDefaults>) {
-        ConfigureUtil.configureSelf(configure, defaults)
-    }
-
     fun configurations(configureClosure: Closure<NamedDomainObjectContainer<BenchmarkConfiguration>>): NamedDomainObjectContainer<BenchmarkConfiguration> {
         return configurations.configure(configureClosure)
     }
 
     val configurations: NamedDomainObjectContainer<BenchmarkConfiguration> = run {
-        project.container(BenchmarkConfiguration::class.java) { name ->
+        val container = project.container(BenchmarkConfiguration::class.java) { name ->
             BenchmarkConfiguration(this, name)
         }
+        container.register("main")
+        container
     }
     
     fun targets(configureClosure: Closure<NamedDomainObjectContainer<BenchmarkTarget>>): NamedDomainObjectContainer<BenchmarkTarget> {
diff --git a/plugin/main/src/org/jetbrains/gradle/benchmarks/JsNodeTasks.kt b/plugin/main/src/org/jetbrains/gradle/benchmarks/JsNodeTasks.kt
@@ -53,9 +53,11 @@ fun Project.createJsBenchmarkExecTask(
 
         arguments("-n", target.name)
         arguments("-r", reportFile.toString())
-        arguments("-i", config.iterations().toString())
-        arguments("-it", config.iterationTime().toString())
-        arguments("-itu", config.iterationTimeUnit.toString())
+        config.iterations?.let { arguments("-i", it.toString()) }
+        config.iterationTime?.let { arguments("-it", it.toString()) }
+        config.iterationTimeUnit?.let { arguments("-itu", it) }
+        config.outputTimeUnit?.let { arguments("-otu", it) }
+        config.mode?.let { arguments("-m", it) }
 
         config.includes.forEach {
             arguments("-I", it)
diff --git a/plugin/main/src/org/jetbrains/gradle/benchmarks/JvmTasks.kt b/plugin/main/src/org/jetbrains/gradle/benchmarks/JvmTasks.kt
@@ -110,9 +110,11 @@ fun Project.createJvmBenchmarkExecTask(
         
         args("-n", target.name)
         args("-r", reportFile.toString())
-        args("-i", config.iterations().toString())
-        args("-it", config.iterationTime().toString())
-        args("-itu", config.iterationTimeUnit.toString())
+        config.iterations?.let { args("-i", it.toString()) }
+        config.iterationTime?.let { args("-it", it.toString()) }
+        config.iterationTimeUnit?.let { args("-itu", it) }
+        config.outputTimeUnit?.let { args("-otu", it) }
+        config.mode?.let { args("-m", it) }
 
         config.includes.forEach {
             args("-I", it)
diff --git a/plugin/main/src/org/jetbrains/gradle/benchmarks/NativeMultiplatformTasks.kt b/plugin/main/src/org/jetbrains/gradle/benchmarks/NativeMultiplatformTasks.kt
@@ -114,9 +114,11 @@ fun Project.createNativeBenchmarkExecTask(
 
         args("-n", target.name)
         args("-r", reportFile.toString())
-        args("-i", config.iterations().toString())
-        args("-it", config.iterationTime().toString())
-        args("-itu", config.iterationTimeUnit.toString())
+        config.iterations?.let { args("-i", it.toString()) }
+        config.iterationTime?.let { args("-it", it.toString()) }
+        config.iterationTimeUnit?.let { args("-itu", it) }
+        config.outputTimeUnit?.let { args("-otu", it) }
+        config.mode?.let { args("-m", it) }
 
         config.includes.forEach {
             args("-I", it)
diff --git a/runtime/commonMain/src/org/jetbrains/gradle/benchmarks/SuiteExecutor.kt b/runtime/commonMain/src/org/jetbrains/gradle/benchmarks/SuiteExecutor.kt
@@ -18,15 +18,18 @@ abstract class SuiteExecutor(val executionName: String, arguments: Array<out Str
     fun run() {
         //println(config.toString())
         reporter.startSuite(executionName)
-        val include = config.include.map { Regex(it) }
+        val include = if (config.include.isEmpty())
+            listOf(Regex(".*"))
+        else
+            config.include.map { Regex(it) }
         val exclude = config.exclude.map { Regex(it) }
 
         @Suppress("UNCHECKED_CAST")
         val benchmarks = suites.flatMap { suite ->
             suite.benchmarks
                 .filter { benchmark ->
                     val fullName = suite.name + "." + benchmark.name
-                    include.any { it.matches(fullName) } && exclude.none { it.matches(fullName) }
+                    include.any { it.containsMatchIn(fullName) } && exclude.none { it.containsMatchIn(fullName) }
                 } as List<BenchmarkDescriptor<Any?>>
         }
 

Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,7 @@ sourceSets {`
`21`	`21`	`// You can add "+ sourceSets.test.output" to include test output as well`
`22`	`22`	`dependencies {`
`23`	`23`	`implementation(project(":runtime"))`
`24`		`- benchmarksCompile sourceSets.main.output + sourceSets.main.compileClasspath`
	`24`	`+ benchmarksCompile sourceSets.main.output + sourceSets.main.runtimeClasspath`
`25`	`25`	`}`
`26`	`26`
`27`	`27`	`// Configure benchmark`