Added nativeGCCollectMode setting to choose strategy of calling K/N GC

Author: Elena Lepilkina

README.md

@@ -148,6 +148,9 @@ Available configuration options:
 * `mode` – "thrpt" for measuring operations per time, or "avgt" for measuring time per operation
 * `nativeIterationMode` – "external" for iterating in gradle in order to get correct Kotlin/Native runtime input in measurement,
  "internal" can be used if it's known that measured code have no calls in K/N runtime that can influence on measurement unrepeatedly.
+* `nativeGCCollectMode`  – "iteration" to collect garbage after each measuring iteration, 
+  "auto"(default) for collecting garbage at the moments choosen by K/N compiler, time for each iteration could be very 
+  different then because of unpredictable call of GC.
 * `include("…")` – regular expression to include benchmarks with fully qualified names matching it, as a substring
 * `exclude("…")` – regular expression to exclude benchmarks with fully qualified names matching it, as a substring
 * `param("name", "value1", "value2")` – specify a parameter for a public mutable property `name` annotated with `@Param`

examples/kotlin-multiplatform/build.gradle

@@ -99,6 +99,7 @@ benchmark {
             iterationTimeUnit = "ms" // time in ms per iteration
             advanced("forks", 1)
             nativeIterationMode = "internal"
+            nativeGCCollectMode = "iteration"
         }
 
         csv {

plugin/main/src/kotlinx/benchmark/gradle/BenchmarkConfiguration.kt

@@ -10,6 +10,7 @@ open class BenchmarkConfiguration(val extension: BenchmarksExtension, val name:
     var iterationTimeUnit: String? = null
     var mode: String? = null
     var nativeIterationMode: String? = null // TODO: where should warning about K/N specific of this parameter be shown?
+    var nativeGCCollectMode: String? = null
     var outputTimeUnit: String? = null
     var reportFormat: String? = null
 

plugin/main/src/kotlinx/benchmark/gradle/Utils.kt

@@ -95,6 +95,7 @@ fun writeParameters(
         config.outputTimeUnit?.let { appendln("outputTimeUnit:$it") }
         config.mode?.let { appendln("mode:$it") }
         config.nativeIterationMode?.let { appendln("nativeIterationMode:$it") }
+        config.nativeGCCollectMode?.let { appendln("nativeGCCollectMode:$it") }
 
         config.includes.forEach {
             appendln("include:$it")

runtime/commonMain/src/kotlinx/benchmark/BenchmarkReportFormatter.kt

@@ -157,12 +157,12 @@ private class CsvBenchmarkReportFormatter(val delimiter: String) : BenchmarkRepo
 
 }
 
-object JsonBenchmarkReportFormatter : BenchmarkReportFormatter() {
+private object JsonBenchmarkReportFormatter : BenchmarkReportFormatter() {
 
     override fun format(results: Collection<ReportBenchmarkResult>): String =
         results.joinToString(",", prefix = "[", postfix = "\n]", transform = this::format)
 
-    fun format(result: ReportBenchmarkResult): String =
+    private fun format(result: ReportBenchmarkResult): String =
         """
   {
     "benchmark" : "${result.benchmark.name}",
@@ -175,6 +175,7 @@ object JsonBenchmarkReportFormatter : BenchmarkReportFormatter() {
           ${result.params.entries.joinToString(separator = ",\n          ") { "\"${it.key}\" : \"${it.value}\"" }}
     },
     "nativeIterationMode" : "${result.config.nativeIterationMode.toText()}",
+    "nativeGCCollectMode" : "${result.config.nativeGCCollectMode.toText()}",
     "primaryMetric" : {
        "score": ${result.score},
        "scoreError": ${result.error},

runtime/commonMain/src/kotlinx/benchmark/CommonBenchmarkAnnotations.kt

@@ -28,6 +28,10 @@ enum class NativeIterationMode {
     Internal, External
 }
 
+enum class NativeGCCollectMode {
+    Auto, Iteration
+}
+
 @Target(AnnotationTarget.CLASS)
 expect annotation class OutputTimeUnit(val value: BenchmarkTimeUnit)
 
@@ -67,6 +71,13 @@ fun NativeIterationMode.toText() = when (this) {
     else -> throw UnsupportedOperationException("$this is not supported")
 }
 
+@Suppress("REDUNDANT_ELSE_IN_WHEN")
+fun NativeGCCollectMode.toText() = when (this) {
+    NativeGCCollectMode.Auto -> "auto"
+    NativeGCCollectMode.Iteration -> "iteration"
+    else -> throw UnsupportedOperationException("$this is not supported")
+}
+
 @Suppress("REDUNDANT_ELSE_IN_WHEN")
 fun BenchmarkTimeUnit.toMultiplier() = when (this) {
     BenchmarkTimeUnit.NANOSECONDS -> 1

runtime/commonMain/src/kotlinx/benchmark/ExecutorConfiguration.kt

@@ -7,7 +7,8 @@ class BenchmarkConfiguration private constructor(
     val iterationTimeUnit: BenchmarkTimeUnit,
     val outputTimeUnit: BenchmarkTimeUnit,
     val mode: Mode,
-    val nativeIterationMode: NativeIterationMode) {
+    val nativeIterationMode: NativeIterationMode,
+    val nativeGCCollectMode: NativeGCCollectMode) {
 
     constructor(runner: RunnerConfiguration, suite: SuiteDescriptor<*>) : this(
         runner.iterations ?: suite.iterations,
@@ -16,13 +17,15 @@ class BenchmarkConfiguration private constructor(
         runner.iterationTimeUnit ?: suite.iterationTime.timeUnit,
         runner.outputTimeUnit ?: suite.outputTimeUnit,
         runner.mode ?: suite.mode,
-        runner.nativeIterationMode ?: NativeIterationMode.External
+        runner.nativeIterationMode ?: NativeIterationMode.External,
+        runner.nativeGCCollectMode ?: NativeGCCollectMode.Auto
     )
 
     override fun toString() =
         "iterations=$iterations, warmups=$warmups, iterationTime=$iterationTime, " +
                 "iterationTimeUnit=${iterationTimeUnit.toText()}, outputTimeUnit=${outputTimeUnit.toText()}, " +
-                "mode=${mode.toText()}, nativeIterationMode=${nativeIterationMode.toText()}"
+                "mode=${mode.toText()}, nativeIterationMode=${nativeIterationMode.toText()}, " +
+                "nativeGCCollectMode=${nativeGCCollectMode.toText()}"
 
     companion object {
         fun parse(description: String): BenchmarkConfiguration {
@@ -38,7 +41,8 @@ class BenchmarkConfiguration private constructor(
                 parseTimeUnit(getParameterValue("iterationTimeUnit")),
                 parseTimeUnit(getParameterValue("outputTimeUnit")),
                 getParameterValue("mode").toMode(),
-                NativeIterationMode.valueOf(getParameterValue("nativeIterationMode").capitalize())
+                NativeIterationMode.valueOf(getParameterValue("nativeIterationMode").capitalize()),
+                NativeGCCollectMode.valueOf(getParameterValue("nativeGCCollectMode").capitalize())
             )
         }
     }

runtime/commonMain/src/kotlinx/benchmark/RunnerConfiguration.kt

@@ -72,6 +72,10 @@ class RunnerConfiguration(config: String) {
         "nativeIterationMode"
     ) { NativeIterationMode.valueOf(it.capitalize()) }
 
+    val nativeGCCollectMode = singleValueOrNull(
+        "nativeGCCollectMode"
+    ) { NativeGCCollectMode.valueOf(it.capitalize()) }
+
     override fun toString(): String {
         return """$name -> $reportFile ($traceFormat, $reportFormat)
 params: ${params.entries.joinToString(prefix = "{", postfix = "}") { "${it.key}: ${it.value}" }}
@@ -84,6 +88,7 @@ iterationTimeUnit: $iterationTimeUnit
 outputTimeUnit: $outputTimeUnit            
 mode: $mode
 nativeIterationMode: $nativeIterationMode
+nativeGCCollectMode: $nativeGCCollectMode
 """
     }
 }

runtime/nativeMain/src/kotlinx/benchmark/native/NativeExecutor.kt

@@ -104,7 +104,7 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
             // Execute warmup
             val cycles = externalCyclesNumber ?: warmup(suite.name, benchmarkRun.config, instance, benchmark)
             DoubleArray(iterations) { iteration ->
-                val nanosecondsPerOperation = measure(instance, benchmark, cycles)
+                val nanosecondsPerOperation = measure(instance, benchmark, cycles, benchmarkRun.config.nativeGCCollectMode)
                 val text = nanosecondsPerOperation.nanosToText(benchmarkRun.config.mode, benchmarkRun.config.outputTimeUnit)
                 val iterationNumber = externalIterationNumber ?: iteration
                 reporter.output(
@@ -204,7 +204,8 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
     private fun <T> measure(
         instance: T,
         benchmark: BenchmarkDescriptor<T>,
-        cycles: Int
+        cycles: Int,
+        nativeGCCollectMode: NativeGCCollectMode
     ): Double {
         val executeFunction = benchmark.function
         var counter = cycles
@@ -214,7 +215,8 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
             @Suppress("UNUSED_VARIABLE")
             val result = instance.executeFunction() // ignore result for now, but might need to consume it somehow
         }
-        GC.collect()
+        if (nativeGCCollectMode == NativeGCCollectMode.Iteration)
+            GC.collect()
         val endTime = getTimeNanos()
         val time = endTime - startTime
         return time.toDouble() / cycles
@@ -241,7 +243,8 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
                 endTime = getTimeNanos()
                 iterations++
             }
-            GC.collect()
+            if (config.nativeGCCollectMode == NativeGCCollectMode.Iteration)
+                GC.collect()
             val time = endTime - startTime
             val metric = time.toDouble() / iterations // TODO: metric
             val sample = metric.nanosToText(config.mode, config.outputTimeUnit)