Fix IntelliJ progress reporting in Native

Abduqodiri Qurbonzoda · qurbonzoda · commit a95f76717a90 · 2021-10-04T17:05:44.000+03:00
diff --git a/plugin/build.gradle b/plugin/build.gradle
@@ -65,6 +65,10 @@ sourceSets {
     }
 }
 
+compileKotlin {
+    kotlinOptions.freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn"
+}
+
 dependencies {
     compile group: 'org.jetbrains.kotlin', name: 'kotlin-reflect', version: kotlin_version
 
diff --git a/plugin/main/src/kotlinx/benchmark/gradle/NativeMultiplatformTasks.kt b/plugin/main/src/kotlinx/benchmark/gradle/NativeMultiplatformTasks.kt
@@ -5,8 +5,8 @@ import org.gradle.api.tasks.*
 import org.jetbrains.kotlin.gradle.plugin.mpp.*
 import org.jetbrains.kotlin.konan.target.*
 import java.io.File
-import kotlin.io.path.ExperimentalPathApi
-import kotlin.io.path.createTempFile
+import java.nio.file.Path
+import kotlin.io.path.*
 
 fun Project.processNativeCompilation(target: NativeBenchmarkTarget) {
     val compilation = target.compilation
@@ -97,6 +97,7 @@ private fun Project.createNativeBenchmarkCompileTask(target: NativeBenchmarkTarg
     return benchmarkCompilation
 }
 
+@OptIn(ExperimentalPathApi::class)
 fun Project.createNativeBenchmarkExecTask(
     config: BenchmarkConfiguration,
     target: NativeBenchmarkTarget,
@@ -118,6 +119,7 @@ fun Project.createNativeBenchmarkExecTask(
         executable = executableFile.absolutePath
         this.config = config
         this.workingDir = target.workingDir?.let { File(it) }
+        this.benchProgressPath = createTempFile("bench", ".txt").absolutePath
 
         onlyIf { executableFile.exists() }
         benchsDescriptionDir = file(project.buildDir.resolve(target.extension.benchsDescriptionDir).resolve(config.name))
@@ -155,6 +157,9 @@ open class NativeBenchmarkExec() : DefaultTask() {
     @Input
     lateinit var benchsDescriptionDir: File
 
+    @Input
+    lateinit var benchProgressPath: String
+
     private fun execute(args: Collection<String>) {
         project.exec {
             it.executable = executable
@@ -164,11 +169,11 @@ open class NativeBenchmarkExec() : DefaultTask() {
         }
     }
 
-    @ExperimentalPathApi
+    @OptIn(ExperimentalPathApi::class)
     @TaskAction
     fun run() {
         // Get full list of running benchmarks
-        execute(listOf(configFile.absolutePath, "--list", benchsDescriptionDir.absolutePath))
+        execute(listOf(configFile.absolutePath, "--list", benchProgressPath, benchsDescriptionDir.absolutePath))
         val detailedConfigFiles = project.fileTree(benchsDescriptionDir).files.sortedBy { it.absolutePath }
         val runResults = mutableMapOf<String, String>()
 
@@ -180,8 +185,8 @@ open class NativeBenchmarkExec() : DefaultTask() {
 
             // Execute benchmark
             if (config.nativeIterationMode == "internal") {
-                val suiteResultsFile = createTempFile("bench", ".txt").toFile()
-                execute(listOf(configFile.absolutePath, "--internal", runConfigPath, suiteResultsFile.absolutePath))
+                val suiteResultsFile = createTempFile("bench", ".txt")
+                execute(listOf(configFile.absolutePath, "--internal", benchProgressPath, runConfigPath, suiteResultsFile.absolutePath))
                 val suiteResults = suiteResultsFile.readText()
                 if (suiteResults.isNotEmpty())
                     runResults[runConfigPath] = suiteResults
@@ -192,10 +197,10 @@ open class NativeBenchmarkExec() : DefaultTask() {
                     .substringBefore(',').toInt()
                 // Warm up
                 var exceptionDuringExecution = false
-                var textResult: File? = null
+                var textResult: Path? = null
                 for (i in 0 until warmups) {
-                    textResult = createTempFile("bench", ".txt").toFile()
-                    execute(listOf(configFile.absolutePath, "--warmup", runConfigPath, i.toString(), textResult.absolutePath))
+                    textResult = createTempFile("bench", ".txt")
+                    execute(listOf(configFile.absolutePath, "--warmup", benchProgressPath, runConfigPath, i.toString(), textResult.absolutePath))
                     val result = textResult.readLines().getOrNull(0)
                     if (result == "null") {
                         exceptionDuringExecution = true
@@ -208,9 +213,9 @@ open class NativeBenchmarkExec() : DefaultTask() {
                 val iterationResults = mutableListOf<Double>()
                 var iteration = 0
                 while (!exceptionDuringExecution && iteration in 0 until iterations) {
-                    val textResult = createTempFile("bench", ".txt").toFile()
+                    textResult = createTempFile("bench", ".txt")
                     execute(
-                        listOf(configFile.absolutePath, "--iteration", runConfigPath, iteration.toString(),
+                        listOf(configFile.absolutePath, "--iteration", benchProgressPath, runConfigPath, iteration.toString(),
                             cycles, textResult.absolutePath)
                     )
                     val result = textResult.readLines()[0]
@@ -221,22 +226,22 @@ open class NativeBenchmarkExec() : DefaultTask() {
                 }
                 // Store results
                 if (iterationResults.size == iterations) {
-                    val iterationsResultsFile = createTempFile("bench_results").toFile()
-                    iterationsResultsFile.printWriter().use { out ->
+                    val iterationsResultsFile = createTempFile("bench_results")
+                    iterationsResultsFile.bufferedWriter().use { out ->
                         out.write(iterationResults.joinToString { it.toString() })
                     }
                     execute(
-                        listOf(configFile.absolutePath, "--end-run", runConfigPath, iterationsResultsFile.absolutePath)
+                        listOf(configFile.absolutePath, "--end-run", benchProgressPath, runConfigPath, iterationsResultsFile.absolutePath)
                     )
                     runResults[runConfigPath] = iterationResults.joinToString()
                 }
             }
         }
         // Merge results
-        val samplesFile = createTempFile("bench_results").toFile()
-        samplesFile.printWriter().use { out ->
+        val samplesFile = createTempFile("bench_results")
+        samplesFile.bufferedWriter().use { out ->
             out.write(runResults.toList().joinToString("\n") { "${it.first}: ${it.second}"})
         }
-        execute(listOf(configFile.absolutePath, "--store-results", samplesFile.absolutePath))
+        execute(listOf(configFile.absolutePath, "--store-results", benchProgressPath, samplesFile.absolutePath))
     }
 }
diff --git a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt
@@ -6,10 +6,12 @@ import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.invocation.Gradle
 import org.gradle.api.tasks.TaskProvider
-import org.gradle.util.GradleVersion
 import java.io.File
+import java.nio.file.Path
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import kotlin.io.path.ExperimentalPathApi
+import kotlin.io.path.invariantSeparatorsPath
 
 fun cleanup(file: File) {
     if (file.exists()) {
@@ -91,6 +93,9 @@ fun Task.traceFormat(): String {
     return if (ideaActive) "xml" else "text"
 }
 
+@OptIn(ExperimentalPathApi::class)
+val Path.absolutePath: String get() = toAbsolutePath().invariantSeparatorsPath
+
 fun writeParameters(
     name: String,
     reportFile: File,
diff --git a/runtime/commonMain/src/kotlinx/benchmark/BenchmarkProgress.kt b/runtime/commonMain/src/kotlinx/benchmark/BenchmarkProgress.kt
@@ -10,9 +10,9 @@ abstract class BenchmarkProgress {
     abstract fun output(suite: String, benchmark: String, message: String)
 
     companion object {
-        fun create(format: String) = when (format) {
+        fun create(format: String, xml: (() -> BenchmarkProgress)? = null) = when (format) {
             "xml" -> {
-                IntelliJBenchmarkProgress()
+                xml?.invoke() ?: IntelliJBenchmarkProgress()
             }
             "text" -> {
                 ConsoleBenchmarkProgress()
@@ -27,7 +27,7 @@ abstract class BenchmarkProgress {
     }
 }
 
-class IntelliJBenchmarkProgress : BenchmarkProgress() {
+open class IntelliJBenchmarkProgress : BenchmarkProgress() {
     private val rootId = "[root]"
 
     override fun startSuite(suite: String) {
@@ -45,9 +45,9 @@ class IntelliJBenchmarkProgress : BenchmarkProgress() {
         println(ijSuiteFinish("", rootId, suiteStatus))
     }
 
-    private var currentClass = ""
-    private var currentStatus = FinishStatus.Success
-    private var suiteStatus = FinishStatus.Success
+    protected var currentClass = ""
+    protected var currentStatus = FinishStatus.Success
+    protected var suiteStatus = FinishStatus.Success
 
     override fun startBenchmark(suite: String, benchmark: String) {
         val methodName = benchmark.substringAfterLast('.')
diff --git a/runtime/commonMain/src/kotlinx/benchmark/SuiteExecutor.kt b/runtime/commonMain/src/kotlinx/benchmark/SuiteExecutor.kt
@@ -1,11 +1,13 @@
 package kotlinx.benchmark
 
-abstract class SuiteExecutor(val executionName: String, arguments: Array<out String>) {
-    private val config = RunnerConfiguration(arguments.first().readFile())
-
-    protected val additionalArguments = arguments.drop(1)
+abstract class SuiteExecutor(
+    val executionName: String,
+    configPath: String,
+    xmlReporter: (() -> BenchmarkProgress)? = null
+) {
+    private val config = RunnerConfiguration(configPath.readFile())
 
-    val reporter = BenchmarkProgress.create(config.traceFormat)
+    val reporter = BenchmarkProgress.create(config.traceFormat, xmlReporter)
 
     private val reportFormatter = BenchmarkReportFormatter.create(config.reportFormat)
 
diff --git a/runtime/jsMain/src/kotlinx/benchmark/js/JsExecutor.kt b/runtime/jsMain/src/kotlinx/benchmark/js/JsExecutor.kt
@@ -4,7 +4,7 @@ import kotlinx.benchmark.*
 import kotlin.js.Promise
 
 class JsExecutor(name: String, @Suppress("UNUSED_PARAMETER") dummy_args: Array<out String>) :
-    SuiteExecutor(name, (process["argv"] as Array<String>).drop(2).toTypedArray()) {
+    SuiteExecutor(name, (process["argv"] as Array<String>)[2]) {
     
     private val benchmarkJs: dynamic = require("benchmark")
 
diff --git a/runtime/nativeMain/src/kotlinx/benchmark/NativeIntelliJBenchmarkProgress.kt b/runtime/nativeMain/src/kotlinx/benchmark/NativeIntelliJBenchmarkProgress.kt
@@ -0,0 +1,38 @@
+package kotlinx.benchmark
+
+// This implementation is used in Kotlin/Native target, where each benchmark is run in a separate process.
+// Previous benchmark id is stored to identify when previous suite (benchmark class) ends.
+class NativeIntelliJBenchmarkProgress(
+    private val benchProgressPath: String
+) : IntelliJBenchmarkProgress() {
+
+    init {
+        decode()
+    }
+
+    override fun startBenchmark(suite: String, benchmark: String) {
+        super.startBenchmark(suite, benchmark)
+        encode()
+    }
+
+    override fun endBenchmarkException(suite: String, benchmark: String, error: String, stacktrace: String) {
+        super.endBenchmarkException(suite, benchmark, error, stacktrace)
+        encode()
+    }
+
+    private fun encode() {
+        writeFile(benchProgressPath, listOf(currentClass, currentStatus, suiteStatus).joinToString(separator = "\n"))
+    }
+
+    private fun decode() {
+        val content = benchProgressPath.readFile()
+        if (content.isEmpty()) {
+            return
+        }
+
+        val (currentClass, currentStatus, suiteStatus) = content.lines()
+        this.currentClass = currentClass
+        this.currentStatus = FinishStatus.valueOf(currentStatus)
+        this.suiteStatus = FinishStatus.valueOf(suiteStatus)
+    }
+}
diff --git a/runtime/nativeMain/src/kotlinx/benchmark/Utils.kt b/runtime/nativeMain/src/kotlinx/benchmark/Utils.kt
@@ -43,9 +43,9 @@ actual fun String.readFile(): String = buildString {
             while (true) {
                 val bufferLength = 64 * 1024
                 val buffer = allocArray<ByteVar>(bufferLength)
-                val line = fgets(buffer, bufferLength, file)?.toKString()
+                val line = fgets(buffer, bufferLength, file)?.toKString() // newline symbol is included
                 if (line == null || line.isEmpty()) break
-                appendLine(line)
+                append(line)
             }
         }
 
diff --git a/runtime/nativeMain/src/kotlinx/benchmark/native/NativeExecutor.kt b/runtime/nativeMain/src/kotlinx/benchmark/native/NativeExecutor.kt
@@ -4,7 +4,13 @@ import kotlinx.benchmark.*
 import kotlin.native.internal.GC
 import kotlin.system.*
 
-class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name, args) {
+class NativeExecutor(
+    name: String,
+    args: Array<out String>
+) : SuiteExecutor(name, args[0], { NativeIntelliJBenchmarkProgress(args[2]) }) {
+
+    private val action = args[1]
+    private val additionalArguments = args.drop(3)
 
     data class BenchmarkRun(val benchmarkName: String, val config: BenchmarkConfiguration,
                             val parameters: Map<String, String>)
@@ -23,7 +29,7 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
                     appendLine("configuration: $config")
                     appendLine("parameters: $params")
                 }
-                val fileName = "${additionalArguments[1]}/${suite.name}_${it.name}_$parametersId.txt"
+                val fileName = "${additionalArguments[0]}/${suite.name}_${it.name}_$parametersId.txt"
                 writeFile(fileName, benchmarkRunConfig)
                 parametersId++
             }
@@ -34,15 +40,15 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
         ?: throw NoSuchElementException("Benchmark $name wasn't found.")
 
     private fun runBenchmarkIteration(benchmarks: List<BenchmarkDescriptor<Any?>>) {
-        val (_, configFileName, iteration, cycles, resultsFile) = additionalArguments
+        val (configFileName, iteration, cycles, resultsFile) = additionalArguments
         val benchmarkRun = configFileName.parseBenchmarkConfig()
         val benchmark = benchmarks.getBenchmark(benchmarkRun.benchmarkName)
         val samples = run(benchmark, benchmarkRun, iteration.toInt(), cycles.toInt())
         writeFile(resultsFile, samples?.let{ it[0].toString() } ?: "null")
     }
 
     private fun runBenchmarkWarmup(benchmarks: List<BenchmarkDescriptor<Any?>>) {
-        val (_, configFileName, iteration, resultsFile) = additionalArguments
+        val (configFileName, iteration, resultsFile) = additionalArguments
         val benchmarkRun = configFileName.parseBenchmarkConfig()
         val benchmark = benchmarks.getBenchmark(benchmarkRun.benchmarkName)
         val id = id(benchmark.name, benchmarkRun.parameters)
@@ -69,7 +75,7 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
     }
 
     private fun runBenchmark(benchmarks: List<BenchmarkDescriptor<Any?>>) {
-        val (_, configFileName, resultsFile) = additionalArguments
+        val (configFileName, resultsFile) = additionalArguments
         val benchmarkRun = configFileName.parseBenchmarkConfig()
         val benchmark = benchmarks.getBenchmark(benchmarkRun.benchmarkName)
         val id = id(benchmark.name, benchmarkRun.parameters)
@@ -82,7 +88,7 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
     }
 
     private fun endExternalBenchmarksRun(benchmarks: List<BenchmarkDescriptor<Any?>>) {
-        val (_, configFileName, samplesFile) = additionalArguments
+        val (configFileName, samplesFile) = additionalArguments
         val samples = samplesFile.readFile().split(", ").map { it.toDouble() }.toDoubleArray()
         val benchmarkRun = configFileName.parseBenchmarkConfig()
         val benchmark = benchmarks.getBenchmark(benchmarkRun.benchmarkName)
@@ -153,8 +159,7 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
     }
 
     private fun storeResults(benchmarks: List<BenchmarkDescriptor<Any?>>, complete: () -> Unit) {
-        val (_, resultsFileName) = additionalArguments
-        resultsFileName.readFile().split("\n").filter { it.isNotEmpty() }.forEach {
+        additionalArguments[0].readFile().lines().forEach {
             val (configFileName, samplesList) = it.split(": ")
             val samples = samplesList.split(", ").map { it.toDouble() }.toDoubleArray()
             val benchmarkRun = configFileName.parseBenchmarkConfig()
@@ -172,26 +177,14 @@ class NativeExecutor(name: String, args: Array<out String>) : SuiteExecutor(name
         start: () -> Unit,
         complete: () -> Unit
     ) {
-        val knownActions = mapOf(
-            "--list" to 2,
-            "--store-results" to 2,
-            "--end-run" to 3,
-            "--internal" to 3,
-            "--iteration" to 5,
-            "--warmup" to 4
-        )
-
-        val action = additionalArguments.first()
-        if (action !in knownActions.keys)
-            throw NoSuchElementException("Action $action isn't found in the list of possible actions ${knownActions.keys}.")
-        require(knownActions[action] == additionalArguments.size)
         when (action) {
             "--list" -> outputBenchmarks(runnerConfiguration, benchmarks, start)
             "--store-results" -> storeResults(benchmarks, complete)
             "--internal" -> runBenchmark(benchmarks)
             "--iteration" -> runBenchmarkIteration(benchmarks)
             "--warmup" -> runBenchmarkWarmup(benchmarks)
             "--end-run" -> endExternalBenchmarksRun(benchmarks)
+            else -> throw IllegalArgumentException("Unknown action: $action.")
         }
     }
     

Original file line number	Diff line number	Diff line change
`@@ -65,6 +65,10 @@ sourceSets {`
`65`	`65`	`}`
`66`	`66`	`}`
`67`	`67`
	`68`	`+compileKotlin {`
	`69`	`+ kotlinOptions.freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn"`
	`70`	`+}`
	`71`	`+`
`68`	`72`	`dependencies {`
`69`	`73`	`compile group: 'org.jetbrains.kotlin', name: 'kotlin-reflect', version: kotlin_version`
`70`	`74`
Original file line number	Diff line number	Diff line change
`@@ -43,9 +43,9 @@ actual fun String.readFile(): String = buildString {`
`43`	`43`	`while (true) {`
`44`	`44`	`val bufferLength = 64 * 1024`
`45`	`45`	`val buffer = allocArray<ByteVar>(bufferLength)`
`46`		`- val line = fgets(buffer, bufferLength, file)?.toKString()`
	`46`	`+ val line = fgets(buffer, bufferLength, file)?.toKString() // newline symbol is included`
`47`	`47`	`if (line == null \|\| line.isEmpty()) break`
`48`		`- appendLine(line)`
	`48`	`+ append(line)`
`49`	`49`	`}`
`50`	`50`	`}`
`51`	`51`