Use "internal" native invocation mode by default

Author: Abduqodiri Qurbonzoda

README.md

@@ -145,12 +145,16 @@ Available configuration options:
 * `iterationTime` – time to run each iteration (measuring and warmup)
 * `iterationTimeUnit` – time unit for `iterationTime` (default is seconds)
 * `outputTimeUnit` – time unit for results output
-* `mode` – "thrpt" for measuring operations per time, or "avgt" for measuring time per operation
-* `nativeIterationMode` – "external" for iterating in gradle in order to get correct Kotlin/Native runtime input in measurement,
- "internal" can be used if it's known that measured code have no calls in K/N runtime that can influence on measurement unrepeatedly.
-* `nativeGCCollectMode`  – "iteration" to collect garbage after each measuring iteration, 
-  "auto"(default) for collecting garbage at the moments choosen by K/N compiler, time for each iteration could be very 
-  different then because of unpredictable call of GC.
+* `mode`
+  - "thrpt" (default) – measures number of benchmark function invocations per time
+  - "avgt" – measures time per benchmark function invocation
+* `nativeIterationMode`
+  - "internal" (default) – executes all iterations of a benchmark in the same process (one binary execution)
+  - "external" – executes each iteration of a benchmark in a separate process, measures in cold Kotlin/Native runtime environment
+* `nativeGCCollectMode`
+  - "auto" (default) – collects garbage at the moments choosen by K/N compiler, time for each iteration can be very
+    different because of unpredictable GC calls.
+  - "iteration" – collects garbage after each measuring iteration
 * `include("…")` – regular expression to include benchmarks with fully qualified names matching it, as a substring
 * `exclude("…")` – regular expression to exclude benchmarks with fully qualified names matching it, as a substring
 * `param("name", "value1", "value2")` – specify a parameter for a public mutable property `name` annotated with `@Param`

examples/kotlin-multiplatform/build.gradle

@@ -79,7 +79,7 @@ benchmark {
             iterationTime = 300 // time in ms per iteration
             iterationTimeUnit = "ms" // time in ms per iteration
             advanced("forks", 1)
-            advanced("nativeIterationMode", "internal")
+            advanced("nativeIterationMode", "external")
             advanced("nativeGCCollectMode", "iteration")
         }
 

plugin/main/src/kotlinx/benchmark/gradle/NativeMultiplatformTasks.kt

@@ -177,14 +177,16 @@ open class NativeBenchmarkExec() : DefaultTask() {
         val detailedConfigFiles = project.fileTree(benchsDescriptionDir).files.sortedBy { it.absolutePath }
         val runResults = mutableMapOf<String, String>()
 
+        val isInternalIterationMode = config.nativeIterationMode.let { it == null || it == "internal" }
+
         detailedConfigFiles.forEach { runConfig ->
             val runConfigPath = runConfig.absolutePath
             val lines = runConfig.readLines()
             require(lines.size > 1) { "Wrong detailed configuration format" }
             val currentConfigDescription = lines[1]
 
             // Execute benchmark
-            if (config.nativeIterationMode == "internal") {
+            if (isInternalIterationMode) {
                 val suiteResultsFile = createTempFile("bench", ".txt")
                 execute(listOf(configFile.absolutePath, "--internal", benchProgressPath, runConfigPath, suiteResultsFile.absolutePath))
                 val suiteResults = suiteResultsFile.readText()

runtime/commonMain/src/kotlinx/benchmark/ExecutorConfiguration.kt

@@ -17,7 +17,7 @@ class BenchmarkConfiguration private constructor(
         runner.iterationTimeUnit ?: suite.iterationTime.timeUnit,
         runner.outputTimeUnit ?: suite.outputTimeUnit,
         runner.mode ?: suite.mode,
-        runner.nativeIterationMode ?: NativeIterationMode.External,
+        runner.nativeIterationMode ?: NativeIterationMode.Internal,
         runner.nativeGCCollectMode ?: NativeGCCollectMode.Auto
     )
 

runtime/nativeMain/src/kotlinx/benchmark/native/NativeExecutor.kt

@@ -97,6 +97,11 @@ class NativeExecutor(
 
     fun run(benchmark: BenchmarkDescriptor<Any?>, benchmarkRun: BenchmarkRun,
             externalIterationNumber: Int? = null, externalCyclesNumber: Int? = null): DoubleArray?  {
+
+        require((externalIterationNumber == null) == (externalCyclesNumber == null))
+        require(benchmarkRun.config.nativeIterationMode == NativeIterationMode.External && externalIterationNumber != null
+                || benchmarkRun.config.nativeIterationMode == NativeIterationMode.Internal && externalIterationNumber == null)
+
         val id = id(benchmark.name, benchmarkRun.parameters)
         val suite = benchmark.suite
 
@@ -222,11 +227,15 @@ class NativeExecutor(
         benchmark: BenchmarkDescriptor<T>,
         currentIteration: Int? = null
     ): Int {
+        require(config.nativeIterationMode == NativeIterationMode.External && currentIteration != null
+                || config.nativeIterationMode == NativeIterationMode.Internal && currentIteration == null)
+
         var iterations = 0
         val warmupIterations = if (config.nativeIterationMode == NativeIterationMode.External) 1 else config.warmups
         repeat(warmupIterations) { iteration ->
             val benchmarkNanos = config.iterationTime * config.iterationTimeUnit.toMultiplier()
             val executeFunction = benchmark.function
+
             if (config.nativeGCCollectMode == NativeGCCollectMode.Iteration)
                 GC.collect()
             val startTime = getTimeNanos()
@@ -239,12 +248,12 @@ class NativeExecutor(
             }
             if (config.nativeGCCollectMode == NativeGCCollectMode.Iteration)
                 GC.collect()
+
             val time = endTime - startTime
             val metric = time.toDouble() / iterations // TODO: metric
             val sample = metric.nanosToText(config.mode, config.outputTimeUnit)
             val iterationNumber = currentIteration ?: iteration
-            if (config.nativeIterationMode == NativeIterationMode.Internal || currentIteration != null)
-                reporter.output(name, benchmark.name, "Warm-up #$iterationNumber: $sample")
+            reporter.output(name, benchmark.name, "Warm-up #$iterationNumber: $sample")
         }
         return iterations
     }