Skip to content

Commit 9f5d386

Browse files
whyolegwhyoleg
authored
Integration testing (#37)
Implement integration testing of plugin + runtime
1 parent 193cbcb commit 9f5d386

File tree

10 files changed

+350
-1
lines changed

10 files changed

+350
-1
lines changed

‎integration/build.gradle.kts

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import org.jetbrains.kotlin.gradle.dsl.*
2+
import org.jetbrains.kotlin.konan.target.*
3+
4+
plugins {
5+
kotlin("jvm")
6+
}
7+
8+
repositories {
9+
mavenCentral()
10+
}
11+
12+
evaluationDependsOn(":kotlinx-benchmark-runtime")
13+
14+
val Gradle.isConfigurationCacheAvailable
15+
get() = try {
16+
val startParameters = gradle.startParameter
17+
startParameters.javaClass.getMethod("isConfigurationCache")
18+
.invoke(startParameters) as? Boolean
19+
} catch (_: Exception) {
20+
null
21+
} ?: false
22+
23+
fun Project.getSystemProperty(key: String): String? {
24+
return if (gradle.isConfigurationCacheAvailable) {
25+
providers.systemProperty(key).forUseAtConfigurationTime().orNull
26+
} else {
27+
System.getProperty(key)
28+
}
29+
}
30+
31+
val nativeTargetName
32+
get() = when {
33+
project.getSystemProperty("idea.active") == "true" -> "native"
34+
HostManager.hostIsLinux -> "linuxX64"
35+
HostManager.hostIsMingw -> "mingwX64"
36+
HostManager.hostIsMac -> "macosX64"
37+
else -> error("Unknown host: ${HostManager.host}")
38+
}
39+
40+
val runtime get() = project(":kotlinx-benchmark-runtime")
41+
val plugin get() = gradle.includedBuild("plugin")
42+
43+
val AbstractArchiveTask.archiveFilePath get() = archiveFile.get().asFile.path
44+
45+
fun artifactsTask(artifact: String) = runtime.tasks.getByName<AbstractArchiveTask>("${artifact}Jar")
46+
fun artifactsTaskNativeKlibs() = runtime.tasks.getByName("compileKotlin${nativeTargetName.capitalize()}")
47+
48+
fun Task.klibs(): String = outputs.files.filter { it.extension == "klib" }.joinToString("\n")
49+
50+
fun IncludedBuild.classpath() = projectDir.resolve("build/createClasspathManifest")
51+
52+
val createClasspathManifest by tasks.registering {
53+
dependsOn(plugin.task(":createClasspathManifest"))
54+
dependsOn(artifactsTask("jvm"))
55+
dependsOn(artifactsTask("js"))
56+
dependsOn(artifactsTask("metadata"))
57+
dependsOn(artifactsTask("${nativeTargetName}Metadata"))
58+
dependsOn(artifactsTaskNativeKlibs())
59+
60+
val outputDir = file("$buildDir/$name")
61+
outputs.dir(outputDir)
62+
doLast {
63+
outputDir.apply {
64+
mkdirs()
65+
resolve("plugin-classpath.txt").writeText(plugin.classpath().resolve("plugin-classpath.txt").readText())
66+
resolve("runtime-metadata.txt").writeText(artifactsTask("metadata").archiveFilePath)
67+
resolve("runtime-jvm.txt").writeText(artifactsTask("jvm").archiveFilePath)
68+
resolve("runtime-js.txt").writeText(artifactsTask("js").archiveFilePath)
69+
resolve("runtime-native-metadata.txt").writeText(artifactsTask("${nativeTargetName}Metadata").archiveFilePath)
70+
resolve("runtime-native.txt").writeText(artifactsTaskNativeKlibs().klibs())
71+
}
72+
}
73+
}
74+
75+
dependencies {
76+
implementation(files(createClasspathManifest))
77+
implementation(gradleTestKit())
78+
79+
testImplementation(kotlin("test-junit"))
80+
}

‎integration/src/main/kotlin/kotlinx/benchmark/integration/BenchmarkConfiguration.kt

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package kotlinx.benchmark.integration
2+
3+
class BenchmarkConfiguration {
4+
var iterations: Int? = null
5+
var warmups: Int? = null
6+
var iterationTime: Long? = null
7+
var iterationTimeUnit: String? = null
8+
var mode: String? = null
9+
var outputTimeUnit: String? = null
10+
var reportFormat: String? = null
11+
12+
fun lines(name: String): List<String> = """
13+
$name {
14+
iterations = $iterations
15+
warmups = $warmups
16+
iterationTime = $iterationTime
17+
iterationTimeUnit = ${iterationTimeUnit?.escape()}
18+
mode = ${mode?.escape()}
19+
outputTimeUnit = ${outputTimeUnit?.escape()}
20+
reportFormat = ${reportFormat?.escape()}
21+
}
22+
""".trimIndent().split("\n")
23+
}
24+
25+
private fun String.escape(): String = "\"$this\""

‎integration/src/main/kotlin/kotlinx/benchmark/integration/ProjectBuilder.kt

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package kotlinx.benchmark.integration
2+
3+
import java.io.*
4+
5+
class ProjectBuilder {
6+
private val configurations = mutableMapOf<String, BenchmarkConfiguration>()
7+
8+
fun configuration(name: String, configuration: BenchmarkConfiguration.() -> Unit = {}) {
9+
configurations[name] = BenchmarkConfiguration().apply(configuration)
10+
}
11+
12+
fun build(original: String): String {
13+
14+
val script =
15+
"""
16+
benchmark {
17+
configurations {
18+
${configurations.flatMap { it.value.lines(it.key) }.joinToString("\n ")}
19+
}
20+
}
21+
""".trimIndent()
22+
23+
return buildScript + "\n\n" + original + "\n\n" + script
24+
}
25+
}
26+
27+
private val buildScript = run {
28+
"""
29+
buildscript {
30+
dependencies {
31+
classpath files(${readFileList("plugin-classpath.txt")})
32+
}
33+
}
34+
35+
apply plugin: 'kotlin-multiplatform'
36+
apply plugin: 'org.jetbrains.kotlinx.benchmark'
37+
38+
repositories {
39+
mavenCentral()
40+
}
41+
42+
def benchmarkRuntimeMetadata = files(${readFileList("runtime-metadata.txt")})
43+
def benchmarkRuntimeJvm = files(${readFileList("runtime-jvm.txt")})
44+
def benchmarkRuntimeJs = files(${readFileList("runtime-js.txt")})
45+
def benchmarkRuntimeNative = files(${readFileList("runtime-native.txt")})
46+
def benchmarkRuntimeNativeMetadata = files(${readFileList("runtime-native-metadata.txt")})
47+
""".trimIndent()
48+
}
49+
50+
private fun readFileList(fileName: String): String {
51+
val resource = ProjectBuilder::class.java.classLoader.getResource(fileName)
52+
?: throw IllegalStateException("Could not find resource '$fileName'")
53+
val files = File(resource.toURI())
54+
.readLines()
55+
.map { File(it).absolutePath.replace("\\", "\\\\") } // escape backslashes in Windows paths
56+
return files.joinToString(", ") { "'$it'" }
57+
}

‎integration/src/main/kotlin/kotlinx/benchmark/integration/Runner.kt

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package kotlinx.benchmark.integration
2+
3+
import org.gradle.testkit.runner.*
4+
import java.io.*
5+
6+
class Runner(
7+
private val projectDir: File,
8+
private val print: Boolean
9+
) {
10+
11+
private fun gradle(vararg tasks: String): GradleRunner =
12+
GradleRunner.create()
13+
.withProjectDir(projectDir)
14+
.withArguments(*(defaultArguments() + tasks))
15+
.run {
16+
if (print) forwardStdOutput(System.out.bufferedWriter()) else this
17+
}
18+
19+
fun run(vararg tasks: String, fn: BuildResult.() -> Unit = {}) {
20+
val gradle = gradle(*tasks)
21+
val buildResult = gradle.build()
22+
buildResult.fn()
23+
}
24+
25+
fun runAndFail(vararg tasks: String, fn: BuildResult.() -> Unit = {}) {
26+
val gradle = gradle(*tasks)
27+
val buildResult = gradle.buildAndFail()
28+
buildResult.fn()
29+
}
30+
31+
private fun defaultArguments(): Array<String> = arrayOf("--stacktrace")
32+
33+
}

‎integration/src/test/kotlin/kotlinx/benchmark/integration/GradleTest.kt

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package kotlinx.benchmark.integration
2+
3+
import org.junit.*
4+
import org.junit.rules.*
5+
import java.io.*
6+
7+
abstract class GradleTest {
8+
@Rule
9+
@JvmField
10+
internal val testProjectDir: TemporaryFolder = TemporaryFolder(File("build/temp").apply { mkdirs() })
11+
private val rootProjectDir: File get() = testProjectDir.root
12+
13+
fun file(path: String): File = rootProjectDir.resolve(path)
14+
15+
fun reports(configuration: String): List<File> {
16+
val folder = file("build/reports/benchmarks/$configuration")
17+
return folder.listFiles().orEmpty().flatMap { it?.listFiles().orEmpty().toList() }
18+
}
19+
20+
fun project(
21+
name: String,
22+
print: Boolean = false,
23+
build: ProjectBuilder.() -> Unit = {}
24+
): Runner {
25+
val builder = ProjectBuilder().apply(build)
26+
templates.resolve(name).copyRecursively(rootProjectDir)
27+
file("build.gradle").modify(builder::build)
28+
file("settings.gradle").writeText("") // empty settings file
29+
return Runner(rootProjectDir, print)
30+
}
31+
}
32+
33+
private val templates = File("src/test/resources/templates")
34+
35+
private fun File.modify(fn: (String) -> String) {
36+
writeText(fn(readText()))
37+
}

‎integration/src/test/kotlin/kotlinx/benchmark/integration/ReportFormatTest.kt

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package kotlinx.benchmark.integration
2+
3+
import java.io.*
4+
import kotlin.test.*
5+
6+
class ReportFormatTest : GradleTest() {
7+
8+
@Test
9+
fun testReportFormatFileNames() {
10+
val formats = listOf(null, "json", "csv", "scsv", "text")
11+
12+
val runner = project("kotlin-multiplatform", true) {
13+
formats.forEach { format ->
14+
configuration(format ?: "jsonDefault") {
15+
iterations = 1
16+
iterationTime = 100
17+
iterationTimeUnit = "ms"
18+
reportFormat = format
19+
}
20+
}
21+
}
22+
23+
formats.forEach {
24+
val name = it ?: "jsonDefault"
25+
val ext = it ?: "json"
26+
runner.run("${name}Benchmark")
27+
val reports = reports(name)
28+
assertEquals(3, reports.size)
29+
assertEquals(setOf("js.$ext", "jvm.$ext", "native.$ext"), reports.map(File::getName).toSet())
30+
}
31+
}
32+
}

‎integration/src/test/resources/templates/kotlin-multiplatform/build.gradle

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
kotlin {
2+
jvm {
3+
compilations.all {
4+
kotlinOptions.jvmTarget = '1.8'
5+
}
6+
}
7+
8+
js {
9+
nodejs()
10+
}
11+
12+
if (org.jetbrains.kotlin.konan.target.HostManager.hostIsLinux) linuxX64('native')
13+
if (org.jetbrains.kotlin.konan.target.HostManager.hostIsMingw) mingwX64('native')
14+
if (org.jetbrains.kotlin.konan.target.HostManager.hostIsMac) macosX64('native')
15+
16+
sourceSets {
17+
commonMain {
18+
dependencies {
19+
implementation(benchmarkRuntimeMetadata)
20+
}
21+
}
22+
jvmMain {
23+
dependencies {
24+
implementation(benchmarkRuntimeJvm)
25+
}
26+
}
27+
jsMain {
28+
dependencies {
29+
implementation(benchmarkRuntimeJs)
30+
}
31+
}
32+
nativeMain {
33+
dependsOn commonMain
34+
dependencies {
35+
implementation(benchmarkRuntimeNative)
36+
implementation(benchmarkRuntimeNativeMetadata)
37+
}
38+
}
39+
}
40+
}
41+
42+
benchmark {
43+
targets {
44+
register("jvm")
45+
register("js")
46+
register("native")
47+
}
48+
}

‎integration/src/test/resources/templates/kotlin-multiplatform/src/commonMain/kotlin/CommonBenchmark.kt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package test
2+
3+
import kotlinx.benchmark.*
4+
import kotlin.math.*
5+
6+
@State(Scope.Benchmark)
7+
@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
8+
@OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS)
9+
@BenchmarkMode(Mode.Throughput)
10+
open class CommonBenchmark {
11+
@Benchmark
12+
open fun mathBenchmark(): Double {
13+
return log(sqrt(3.0) * cos(3.0), 2.0)
14+
}
15+
}

‎plugin/build.gradle

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,30 @@ dependencies {
8282
testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version"
8383
testImplementation gradleTestKit()
8484
testImplementation "junit:junit:$junit_version"
85-
85+
8686
/*
8787
// This is needed for test to get it on classpath and run
8888
runtime "org.jetbrains.kotlin:kotlin-allopen:$kotlin_version"
8989
runtime "org.jetbrains.kotlin:kotlin-compiler-embeddable:$kotlin_version"
9090
*/
9191
}
92+
93+
// comfiguration of classpathes for plugin integration tests
94+
95+
configurations {
96+
testPluginClasspath
97+
}
98+
99+
dependencies {
100+
testPluginClasspath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
101+
}
102+
103+
task createClasspathManifest {
104+
def outputDir = file("$buildDir/$name")
105+
outputs.dir outputDir
106+
107+
doLast {
108+
outputDir.mkdirs()
109+
file("$outputDir/plugin-classpath.txt").text = (sourceSets.main.runtimeClasspath + configurations.testPluginClasspath).join("\n")
110+
}
111+
}

‎settings.gradle

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ includeBuild("plugin")
1313
include "runtime"
1414
project(":runtime").name = 'kotlinx-benchmark-runtime'
1515

16+
include "integration"
17+
1618
include "examples"
1719
include "examples:kotlin-multiplatform"
1820
include "examples:java"

0 commit comments

Comments
 (0)