Skip to content

Commit 75ff25f

Browse files
Elena LepilkinaLepilkinaElena
Elena Lepilkina
authored and
LepilkinaElena
committed
Support K/N runs
1 parent f3d2a67 commit 75ff25f

File tree

15 files changed

+409
-101
lines changed

15 files changed

+409
-101
lines changed

‎examples/kotlin-multiplatform/build.gradle

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,10 +94,11 @@ benchmark {
9494
fast { // --> jvmFastBenchmark
9595
include("Common")
9696
exclude("long")
97-
iterations = 1
97+
iterations = 10
9898
iterationTime = 300 // time in ms per iteration
9999
iterationTimeUnit = "ms" // time in ms per iteration
100100
advanced("forks", 1)
101+
iterationMode = "internal"
101102
}
102103

103104
csv {

‎plugin/main/src/kotlinx/benchmark/gradle/BenchmarkConfiguration.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ open class BenchmarkConfiguration(val extension: BenchmarksExtension, val name:
99
var iterationTime: Long? = null
1010
var iterationTimeUnit: String? = null
1111
var mode: String? = null
12-
var iterationMode: String? = null
12+
var iterationMode: String? = null // TODO: where should warning about K/N specific of this parameter be shown?
1313
var outputTimeUnit: String? = null
1414
var reportFormat: String? = null
1515

‎plugin/main/src/kotlinx/benchmark/gradle/BenchmarksExtension.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ fun Project.benchmark(configure: Action<BenchmarksExtension>) {
1313
open class BenchmarksExtension(val project: Project) {
1414
var buildDir: String = "benchmarks"
1515
var reportsDir: String = "reports/benchmarks"
16+
var configsDir: String = "configs"
1617

1718
val version = BenchmarksPlugin.PLUGIN_VERSION
1819

‎plugin/main/src/kotlinx/benchmark/gradle/NativeMultiplatformTasks.kt

Lines changed: 123 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ import org.gradle.api.*
44
import org.gradle.api.tasks.*
55
import org.jetbrains.kotlin.gradle.plugin.mpp.*
66
import org.jetbrains.kotlin.konan.target.*
7-
import java.io.*
7+
import java.io.File
8+
import kotlin.io.path.ExperimentalPathApi
9+
import kotlin.io.path.createTempFile
810

911
fun Project.processNativeCompilation(target: NativeBenchmarkTarget) {
1012
val compilation = target.compilation
@@ -14,6 +16,8 @@ fun Project.processNativeCompilation(target: NativeBenchmarkTarget) {
1416
}
1517

1618
project.logger.info("Configuring benchmarks for '${target.name}' using Kotlin/Native")
19+
20+
configureMultiplatformNativeCompilation(target, compilation)
1721

1822
createNativeBenchmarkGenerateSourceTask(target)
1923

@@ -114,28 +118,140 @@ fun Project.createNativeBenchmarkExecTask(
114118
onlyIf { linkTask.enabled }
115119

116120
val reportsDir = benchmarkReportsDir(config, target)
117-
val reportFile = reportsDir.resolve("${target.name}.${config.reportFileExt()}")
121+
reportFile = reportsDir.resolve("${target.name}.json")
118122

119123
val executableFile = linkTask.outputFile.get()
120124
executable = executableFile.absolutePath
121-
if (target.workingDir != null)
122-
workingDir = File(target.workingDir)
125+
this.config = config
126+
this.workingDir = target.workingDir?.let { File(it) }
123127

124128
onlyIf { executableFile.exists() }
129+
configDir = file(project.buildDir.resolve(target.extension.configsDir).resolve(config.name))
130+
configDir.mkdirs()
131+
132+
val ideaActive = (extensions.extraProperties.get("idea.internal.test") as? String)?.toBoolean() ?: false
133+
configFile = writeParameters(target.name, reportFile, if (ideaActive) "xml" else "text", config)
125134

126135
dependsOn(linkTask)
127136
doFirst {
128-
val ideaActive = (extensions.extraProperties.get("idea.internal.test") as? String)?.toBoolean() ?: false
129-
args(writeParameters(target.name, reportFile, if (ideaActive) "xml" else "text", config))
130137
reportsDir.mkdirs()
131138
logger.lifecycle("Running '${config.name}' benchmarks for '${target.name}'")
132139
}
133140
}
134141
}
135142

136-
open class NativeBenchmarkExec : Exec() {
143+
open class NativeBenchmarkExec() : DefaultTask() {
137144
/*
138145
@Option(option = "filter", description = "Configures the filter for benchmarks to run.")
139146
var filter: String? = null
140147
*/
148+
@Input
149+
lateinit var executable: String
150+
151+
var workingDir: File? = null
152+
153+
@Input
154+
lateinit var configFile: File
155+
156+
@Input
157+
lateinit var config: BenchmarkConfiguration
158+
159+
@Input
160+
lateinit var reportFile: File
161+
162+
@Input
163+
lateinit var configDir: File
164+
165+
private fun execute(args: Collection<String>) {
166+
project.exec {
167+
it.executable = executable
168+
it.args(args)
169+
if (workingDir != null)
170+
it.workingDir = workingDir
171+
}
172+
}
173+
174+
@ExperimentalPathApi
175+
@TaskAction
176+
fun run() {
177+
// Get full list of running benchmarks
178+
execute(listOf(configFile.absolutePath, "--list", configDir.absolutePath))
179+
val detailedConfigFiles = project.fileTree(configDir).files.sortedBy { it.absolutePath }
180+
val jsonReportParts = mutableListOf<File>()
181+
182+
detailedConfigFiles.forEach { runConfig ->
183+
val runConfigPath = runConfig.absolutePath
184+
val lines = runConfig.readLines()
185+
require(lines.size > 1) { "Wrong detailed configuration format" }
186+
val currentConfigDescription = lines[1]
187+
188+
// Execute benchmark
189+
if (config.iterationMode == "internal") {
190+
val jsonFile = createTempFile("bench", ".json").toFile()
191+
jsonReportParts.add(jsonFile)
192+
execute(listOf(configFile.absolutePath, "--internal", runConfigPath, jsonFile.absolutePath))
193+
} else {
194+
val iterations = currentConfigDescription.substringAfter("iterations=")
195+
.substringBefore(',').toInt()
196+
val warmups = currentConfigDescription.substringAfter("warmups=")
197+
.substringBefore(',').toInt()
198+
// Warm up
199+
var exceptionDuringExecution = false
200+
for(i in 0 until warmups) {
201+
val textResult = createTempFile("bench", ".txt").toFile()
202+
execute(listOf(configFile.absolutePath, "--warmup", runConfigPath, i.toString(), textResult.absolutePath))
203+
val result = textResult.readLines().getOrNull(0)
204+
if (result == "null") {
205+
exceptionDuringExecution = true
206+
break
207+
}
208+
}
209+
// Execution
210+
val iterationResults = mutableListOf<Double>()
211+
var iteration = 0
212+
while (!exceptionDuringExecution && iteration in 0 until iterations) {
213+
val textResult = createTempFile("bench", ".txt").toFile()
214+
execute(listOf(configFile.absolutePath, runConfigPath, iteration.toString(), textResult.absolutePath))
215+
val result = textResult.readLines()[0]
216+
if (result == "null")
217+
exceptionDuringExecution = true
218+
iterationResults.add(result.toDouble())
219+
iteration++
220+
}
221+
// Store results
222+
if (iterationResults.size == iterations) {
223+
val samplesFile = createTempFile("bench_results").toFile()
224+
samplesFile.printWriter().use { out ->
225+
out.write(iterationResults.joinToString { it.toString() })
226+
}
227+
execute(listOf(configFile.absolutePath, runConfigPath, samplesFile.absolutePath))
228+
jsonReportParts.add(samplesFile)
229+
}
230+
}
231+
}
232+
// Complete
233+
execute(listOf(configFile.absolutePath, "--complete"))
234+
// Merge reports
235+
val fullResults = jsonReportParts.map {
236+
it.readText()
237+
}.joinToString(",", prefix = "[", postfix = "\n]")
238+
reportFile.printWriter().use {
239+
it.print(fullResults)
240+
}
241+
}
242+
}
243+
244+
private fun Project.configureMultiplatformNativeCompilation(
245+
target: NativeBenchmarkTarget,
246+
compilation: KotlinNativeCompilation
247+
) {
248+
val konanTarget = compilation.target.konanTarget
249+
250+
// Add runtime library as an implementation dependency to the specified compilation
251+
val runtime =
252+
dependencies.create("${BenchmarksPlugin.RUNTIME_DEPENDENCY_BASE}-${konanTarget.presetName}:${target.extension.version}")
253+
254+
compilation.dependencies {
255+
//implementation(runtime)
256+
}
141257
}

‎runtime/commonMain/src/kotlinx/benchmark/CommonBenchmarkAnnotations.kt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,14 @@ fun BenchmarkTimeUnit.toText() = when (this) {
4545
else -> throw UnsupportedOperationException("$this is not supported")
4646
}
4747

48+
fun String.toMode() =
49+
when (this) {
50+
"thrpt" -> Mode.Throughput
51+
"avgt" -> Mode.AverageTime
52+
else -> throw UnsupportedOperationException("$this is not supported")
53+
}
54+
55+
4856
@Suppress("REDUNDANT_ELSE_IN_WHEN")
4957
fun Mode.toText() = when (this) {
5058
Mode.Throughput -> "thrpt"

‎runtime/commonMain/src/kotlinx/benchmark/ExecutorConfiguration.kt

Lines changed: 51 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,53 @@
11
package kotlinx.benchmark
22

3-
class BenchmarkConfiguration(val runner: RunnerConfiguration, val suite: SuiteDescriptor<*>) {
4-
val iterations: Int get() = runner.iterations ?: suite.iterations
5-
val warmups: Int get() = runner.warmups ?: suite.warmups
6-
val iterationTime: Long get() = runner.iterationTime ?: suite.iterationTime.value
7-
val iterationTimeUnit: BenchmarkTimeUnit get() = runner.iterationTimeUnit ?: suite.iterationTime.timeUnit
8-
val outputTimeUnit: BenchmarkTimeUnit get() = runner.outputTimeUnit ?: suite.outputTimeUnit
9-
val mode: Mode get() = runner.mode ?: suite.mode
10-
val iterationMode: IterationMode = runner.iterationMode ?: IterationMode.External
11-
}
3+
class BenchmarkConfiguration private constructor(
4+
val iterations: Int,
5+
val warmups: Int,
6+
val iterationTime: Long,
7+
val iterationTimeUnit: BenchmarkTimeUnit,
8+
val outputTimeUnit: BenchmarkTimeUnit,
9+
val mode: Mode,
10+
val iterationMode: IterationMode) {
11+
12+
constructor(runner: RunnerConfiguration, suite: SuiteDescriptor<*>) : this(
13+
runner.iterations ?: suite.iterations,
14+
runner.warmups ?: suite.warmups,
15+
runner.iterationTime ?: suite.iterationTime.value,
16+
runner.iterationTimeUnit ?: suite.iterationTime.timeUnit,
17+
runner.outputTimeUnit ?: suite.outputTimeUnit,
18+
runner.mode ?: suite.mode,
19+
runner.iterationMode ?: IterationMode.External
20+
)
21+
22+
override fun toString() =
23+
"iterations=$iterations, warmups=$warmups, iterationTime=$iterationTime, " +
24+
"iterationTimeUnit=${iterationTimeUnit.toText()}, outputTimeUnit=${outputTimeUnit.toText()}, " +
25+
"mode=${mode.toText()}, iterationMode=${iterationMode.toText()}"
26+
27+
companion object {
28+
fun parse(description: String): BenchmarkConfiguration {
29+
val parameters = description.parseMap()
30+
31+
fun getParameterValue(key: String) =
32+
parameters[key] ?: throw NoSuchElementException("Parameter `$key` is required.")
33+
34+
return BenchmarkConfiguration(
35+
getParameterValue("iterations").toInt(),
36+
getParameterValue("warmups").toInt(),
37+
getParameterValue("iterationTime").toLong(),
38+
parseTimeUnit(getParameterValue("iterationTimeUnit")),
39+
parseTimeUnit(getParameterValue("outputTimeUnit")),
40+
getParameterValue("mode").toMode(),
41+
IterationMode.valueOf(getParameterValue("iterationMode").capitalize())
42+
)
43+
}
44+
}
45+
}
46+
47+
internal fun String.parseMap(): Map<String, String> =
48+
this.removeSurrounding("{", "}").split(", ").filter{ it.isNotEmpty() }.map {
49+
val keyValue = it.split("=")
50+
require(keyValue.size == 2) { "Wrong format of map string description!" }
51+
val (key, value) = keyValue
52+
key to value
53+
}.toMap()

‎runtime/commonMain/src/kotlinx/benchmark/RunnerConfiguration.kt

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -70,17 +70,7 @@ class RunnerConfiguration(config: String) {
7070

7171
val iterationMode = singleValueOrNull(
7272
"iterationMode"
73-
) { IterationMode.valueOf(it) }
74-
75-
76-
private fun parseTimeUnit(text: String) = when (text) {
77-
BenchmarkTimeUnit.SECONDS.name, "s", "sec" -> BenchmarkTimeUnit.SECONDS
78-
BenchmarkTimeUnit.MICROSECONDS.name, "us", "micros" -> BenchmarkTimeUnit.MICROSECONDS
79-
BenchmarkTimeUnit.MILLISECONDS.name, "ms", "millis" -> BenchmarkTimeUnit.MILLISECONDS
80-
BenchmarkTimeUnit.NANOSECONDS.name, "ns", "nanos" -> BenchmarkTimeUnit.NANOSECONDS
81-
BenchmarkTimeUnit.MINUTES.name, "m", "min" -> BenchmarkTimeUnit.MINUTES
82-
else -> throw UnsupportedOperationException("Unknown time unit: $text")
83-
}
73+
) { IterationMode.valueOf(it.capitalize()) }
8474

8575
override fun toString(): String {
8676
return """$name -> $reportFile ($traceFormat, $reportFormat)
@@ -98,4 +88,13 @@ iterationMode: $iterationMode
9888
}
9989
}
10090

101-
expect fun String.readConfigFile(): String
91+
internal fun parseTimeUnit(text: String) = when (text) {
92+
BenchmarkTimeUnit.SECONDS.name, "s", "sec" -> BenchmarkTimeUnit.SECONDS
93+
BenchmarkTimeUnit.MICROSECONDS.name, "us", "micros" -> BenchmarkTimeUnit.MICROSECONDS
94+
BenchmarkTimeUnit.MILLISECONDS.name, "ms", "millis" -> BenchmarkTimeUnit.MILLISECONDS
95+
BenchmarkTimeUnit.NANOSECONDS.name, "ns", "nanos" -> BenchmarkTimeUnit.NANOSECONDS
96+
BenchmarkTimeUnit.MINUTES.name, "m", "min" -> BenchmarkTimeUnit.MINUTES
97+
else -> throw UnsupportedOperationException("Unknown time unit: $text")
98+
}
99+
100+
expect fun String.readFile(): String

‎runtime/commonMain/src/kotlinx/benchmark/SuiteDescriptor.kt

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
package kotlinx.benchmark
22

3+
object DefaultDescriptorParameters {
4+
val iterations = 3
5+
val warmups = 3
6+
val iterationTime = IterationTime(1, BenchmarkTimeUnit.SECONDS)
7+
}
8+
39
open class SuiteDescriptor<T>(
410
val name: String,
511
val factory: () -> T,
@@ -10,10 +16,10 @@ open class SuiteDescriptor<T>(
1016
val parameters: List<String>,
1117
val defaultParameters: Map<String, List<String>>,
1218

13-
val iterations: Int = 3,
14-
val warmups: Int = 3,
19+
val iterations: Int = DefaultDescriptorParameters.iterations,
20+
val warmups: Int = DefaultDescriptorParameters.warmups,
1521

16-
val iterationTime: IterationTime = IterationTime(1, BenchmarkTimeUnit.SECONDS),
22+
val iterationTime: IterationTime = DefaultDescriptorParameters.iterationTime,
1723
val outputTimeUnit: BenchmarkTimeUnit = BenchmarkTimeUnit.MILLISECONDS,
1824
val mode: Mode = Mode.Throughput
1925
) {

‎runtime/commonMain/src/kotlinx/benchmark/SuiteExecutor.kt

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package kotlinx.benchmark
22

33
abstract class SuiteExecutor(val executionName: String, arguments: Array<out String>) {
4-
private val config = RunnerConfiguration(arguments.first().readConfigFile())
4+
private val config = RunnerConfiguration(arguments.first().readFile())
5+
6+
protected val additionalArguments = arguments.drop(1)
57

68
val reporter = BenchmarkProgress.create(config.traceFormat)
79

@@ -17,7 +19,6 @@ abstract class SuiteExecutor(val executionName: String, arguments: Array<out Str
1719

1820
fun run() {
1921
//println(config.toString())
20-
reporter.startSuite(executionName)
2122
val include = if (config.include.isEmpty())
2223
listOf(Regex(".*"))
2324
else
@@ -33,7 +34,7 @@ abstract class SuiteExecutor(val executionName: String, arguments: Array<out Str
3334
} as List<BenchmarkDescriptor<Any?>>
3435
}
3536

36-
run(config, reporter, benchmarks) {
37+
run(config, reporter, benchmarks, { reporter.startSuite(executionName) }) {
3738
val summary = TextBenchmarkReportFormatter.format(results)
3839
reporter.endSuite(executionName, summary)
3940
saveReport(config.reportFile, reportFormatter.format(results))
@@ -48,6 +49,7 @@ abstract class SuiteExecutor(val executionName: String, arguments: Array<out Str
4849
runnerConfiguration: RunnerConfiguration,
4950
reporter: BenchmarkProgress,
5051
benchmarks: List<BenchmarkDescriptor<Any?>>,
52+
start: () -> Unit,
5153
complete: () -> Unit
5254
)
5355

‎runtime/jsMain/src/kotlinx/benchmark/Utils.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@ actual fun saveReport(reportFile: String, report: String) {
1414
fs.writeFile(reportFile, report) { err -> if (err != null) throw err }
1515
}
1616

17-
actual fun String.readConfigFile(): String {
17+
actual fun String.readFile(): String {
1818
return fs.readFileSync(this, "utf8")
1919
}

0 commit comments

Comments
 (0)