Skip to content

Commit 5acabb3

Browse files
wldehwldeh
authored
Validate config values in the plugin (#126)
1 parent e1816b2 commit 5acabb3

File tree

3 files changed

+348
-3
lines changed

3 files changed

+348
-3
lines changed

‎integration/src/main/kotlin/kotlinx/benchmark/integration/BenchmarkConfiguration.kt

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,27 @@ class BenchmarkConfiguration {
88
var mode: String? = null
99
var outputTimeUnit: String? = null
1010
var reportFormat: String? = null
11+
var includes: MutableList<String> = mutableListOf()
12+
var excludes: MutableList<String> = mutableListOf()
13+
var params: MutableMap<String, MutableList<Any>> = mutableMapOf()
14+
var advanced: MutableMap<String, Any> = mutableMapOf()
15+
16+
fun include(pattern: String) {
17+
includes.add(pattern)
18+
}
19+
20+
fun exclude(pattern: String) {
21+
excludes.add(pattern)
22+
}
23+
24+
fun param(name: String, vararg value: Any) {
25+
val values = params.getOrPut(name) { mutableListOf() }
26+
values.addAll(value)
27+
}
28+
29+
fun advanced(name: String, value: Any) {
30+
advanced[name] = value
31+
}
1132

1233
fun lines(name: String): List<String> = """
1334
$name {
@@ -18,8 +39,14 @@ class BenchmarkConfiguration {
1839
mode = ${mode?.escape()}
1940
outputTimeUnit = ${outputTimeUnit?.escape()}
2041
reportFormat = ${reportFormat?.escape()}
42+
includes = ${includes.map { it.escape() }}
43+
excludes = ${excludes.map { it.escape() }}
44+
${params.entries.joinToString(separator = "\n") { """param("${it.key}", ${it?.value?.joinToString()})""" }}
45+
${advanced.entries.joinToString(separator = "\n") {
46+
"""advanced("${it.key}", ${if (it.value is String) "\"${it.value}\"" else it.value})"""
47+
}}
2148
}
2249
""".trimIndent().split("\n")
2350
}
2451

25-
private fun String.escape(): String = "\"$this\""
52+
private fun String.escape(): String = "\"$this\""

‎integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
package kotlinx.benchmark.integration
2+
3+
import java.io.*
4+
import kotlin.test.*
5+
6+
class OptionsValidationTest : GradleTest() {
7+
8+
@Test
9+
fun testReportFormatValidation() {
10+
val runner = project("kotlin-multiplatform") {
11+
configuration("invalidReportFormat") {
12+
iterations = 1
13+
iterationTime = 100
14+
iterationTimeUnit = "ms"
15+
reportFormat = "htmll"
16+
}
17+
}
18+
19+
runner.runAndFail("invalidReportFormatBenchmark") {
20+
assertOutputContains("Invalid report format: 'htmll'. Accepted formats: json, csv, scsv, text (e.g., reportFormat = 'json').")
21+
}
22+
}
23+
24+
@Test
25+
fun testIterationsValidation() {
26+
val runner = project("kotlin-multiplatform") {
27+
configuration("zeroIterations") {
28+
iterations = 0
29+
iterationTime = 100
30+
iterationTimeUnit = "ms"
31+
}
32+
}
33+
34+
runner.runAndFail("zeroIterationsBenchmark") {
35+
assertOutputContains("Invalid iterations: '0'. Expected a positive integer (e.g., iterations = 5).")
36+
}
37+
}
38+
39+
@Test
40+
fun testWarmupsValidation() {
41+
val runner = project("kotlin-multiplatform") {
42+
configuration("negativeWarmups") {
43+
iterations = 1
44+
warmups = -1
45+
iterationTime = 100
46+
iterationTimeUnit = "ms"
47+
}
48+
}
49+
50+
runner.runAndFail("negativeWarmupsBenchmark") {
51+
assertOutputContains("Invalid warmups: '-1'. Expected a non-negative integer (e.g., warmups = 3).")
52+
}
53+
}
54+
55+
@Test
56+
fun testIterationTimeValidation() {
57+
val runner = project("kotlin-multiplatform") {
58+
configuration("zeroIterationTime") {
59+
iterations = 1
60+
iterationTime = 0
61+
iterationTimeUnit = "ms"
62+
}
63+
}
64+
65+
runner.runAndFail("zeroIterationTimeBenchmark") {
66+
assertOutputContains("Invalid iterationTime: '0'. Expected a positive number (e.g., iterationTime = 300).")
67+
}
68+
}
69+
70+
@Test
71+
fun testIterationTimeUnitValidation() {
72+
val runner = project("kotlin-multiplatform") {
73+
configuration("invalidIterationTimeUnit") {
74+
iterations = 1
75+
iterationTime = 100
76+
iterationTimeUnit = "x"
77+
}
78+
}
79+
80+
runner.runAndFail("invalidIterationTimeUnitBenchmark") {
81+
assertOutputContains("Invalid iterationTimeUnit: 'x'. Accepted units: seconds, s, microseconds, us, milliseconds, ms, nanoseconds, ns, minutes, m (e.g., iterationTimeUnit = 'ms').")
82+
}
83+
}
84+
85+
@Test
86+
fun testModeValidation() {
87+
val runner = project("kotlin-multiplatform") {
88+
configuration("invalidMode") {
89+
iterations = 1
90+
iterationTime = 100
91+
iterationTimeUnit = "ms"
92+
mode = "x"
93+
}
94+
}
95+
96+
runner.runAndFail("invalidModeBenchmark") {
97+
assertOutputContains("Invalid benchmark mode: 'x'. Accepted modes: thrpt, avgt (e.g., mode = 'thrpt').")
98+
}
99+
}
100+
101+
@Test
102+
fun testOutputTimeUnitValidation() {
103+
val runner = project("kotlin-multiplatform") {
104+
configuration("invalidOutputTimeUnit") {
105+
iterations = 1
106+
iterationTime = 100
107+
iterationTimeUnit = "ms"
108+
outputTimeUnit = "x"
109+
}
110+
}
111+
112+
runner.runAndFail("invalidOutputTimeUnitBenchmark") {
113+
assertOutputContains("Invalid outputTimeUnit: 'x'. Accepted units: seconds, s, microseconds, us, milliseconds, ms, nanoseconds, ns, minutes, m (e.g., outputTimeUnit = 'ns').")
114+
}
115+
}
116+
117+
@Test
118+
fun testIncludesValidation() {
119+
val runner = project("kotlin-multiplatform") {
120+
configuration("blankIncludePattern") {
121+
iterations = 1
122+
iterationTime = 100
123+
iterationTimeUnit = "ms"
124+
include(" ")
125+
}
126+
}
127+
128+
runner.runAndFail("blankIncludePatternBenchmark") {
129+
assertOutputContains("Invalid include pattern: ' '. Pattern must not be blank.")
130+
}
131+
}
132+
133+
@Test
134+
fun testExcludesValidation() {
135+
val runner = project("kotlin-multiplatform") {
136+
configuration("blankExcludePattern") {
137+
iterations = 1
138+
iterationTime = 100
139+
iterationTimeUnit = "ms"
140+
exclude(" ")
141+
}
142+
}
143+
144+
runner.runAndFail("blankExcludePatternBenchmark") {
145+
assertOutputContains("Invalid exclude pattern: ' '. Pattern must not be blank.")
146+
}
147+
}
148+
149+
@Test
150+
fun testParamsValidation() {
151+
val runner = project("kotlin-multiplatform") {
152+
configuration("blankParamName") {
153+
iterations = 1
154+
iterationTime = 100
155+
iterationTimeUnit = "ms"
156+
param(" ", 5)
157+
}
158+
}
159+
160+
runner.runAndFail("blankParamNameBenchmark") {
161+
assertOutputContains("Invalid param name: ' '. It must not be blank.")
162+
}
163+
}
164+
165+
@Test
166+
fun testAdvancedValidation() {
167+
val runner = project("kotlin-multiplatform") {
168+
configuration("blankAdvancedConfigName") {
169+
iterations = 1
170+
iterationTime = 100
171+
iterationTimeUnit = "ms"
172+
advanced(" ", "value")
173+
}
174+
175+
configuration("invalidNativeFork") {
176+
iterations = 1
177+
iterationTime = 100
178+
iterationTimeUnit = "ms"
179+
advanced("nativeFork", "x")
180+
}
181+
182+
configuration("invalidNativeGCAfterIteration") {
183+
iterations = 1
184+
iterationTime = 100
185+
iterationTimeUnit = "ms"
186+
advanced("nativeGCAfterIteration", "x")
187+
}
188+
189+
configuration("invalidJvmForks") {
190+
iterations = 1
191+
iterationTime = 100
192+
iterationTimeUnit = "ms"
193+
advanced("jvmForks", "-1")
194+
}
195+
196+
configuration("invalidJsUseBridge") {
197+
iterations = 1
198+
iterationTime = 100
199+
iterationTimeUnit = "ms"
200+
advanced("jsUseBridge", "x")
201+
}
202+
}
203+
204+
runner.runAndFail("blankAdvancedConfigNameBenchmark") {
205+
assertOutputContains("Invalid advanced config name: ' '. It must not be blank.")
206+
}
207+
runner.runAndFail("invalidNativeForkBenchmark") {
208+
assertOutputContains("Invalid value 'x' for 'nativeFork'. Accepted values: 'perBenchmark', 'perIteration' (e.g., nativeFork = 'perBenchmark').")
209+
}
210+
runner.runAndFail("invalidNativeGCAfterIterationBenchmark") {
211+
assertOutputContains("Invalid value 'x' for 'nativeGCAfterIteration'. Expected a boolean value (e.g., nativeGCAfterIteration = true).")
212+
}
213+
runner.runAndFail("invalidJvmForksBenchmark") {
214+
assertOutputContains("Invalid value '-1' for 'jvmForks'. Expected a non-negative integer or 'definedByJmh' (e.g., jvmForks = 2 or jvmForks = 'definedByJmh').")
215+
}
216+
runner.runAndFail("invalidJsUseBridgeBenchmark") {
217+
assertOutputContains("Invalid value 'x' for 'jsUseBridge'. Expected a boolean value (e.g., jsUseBridge = true).")
218+
}
219+
}
220+
}

‎plugin/main/src/kotlinx/benchmark/gradle/Utils.kt

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,11 +134,109 @@ fun writeParameters(
134134

135135
private fun validateConfig(config: BenchmarkConfiguration) {
136136
config.reportFormat?.let {
137-
require(it.toLowerCase() in setOf("json", "csv", "scsv", "text")) {
138-
"Report format '$it' is not supported."
137+
val validFormats = setOf("json", "csv", "scsv", "text")
138+
require(it.toLowerCase() in validFormats) {
139+
"Invalid report format: '$it'. Accepted formats: ${validFormats.joinToString(", ")} (e.g., reportFormat = 'json')."
139140
}
140141
}
141142

143+
config.iterations?.let {
144+
require(it > 0) {
145+
"Invalid iterations: '$it'. Expected a positive integer (e.g., iterations = 5)."
146+
}
147+
}
148+
149+
config.warmups?.let {
150+
require(it >= 0) {
151+
"Invalid warmups: '$it'. Expected a non-negative integer (e.g., warmups = 3)."
152+
}
153+
}
154+
155+
config.iterationTime?.let {
156+
require(it > 0) {
157+
"Invalid iterationTime: '$it'. Expected a positive number (e.g., iterationTime = 300)."
158+
}
159+
require(config.iterationTimeUnit != null) {
160+
"Missing iterationTimeUnit. Please provide iterationTimeUnit when specifying iterationTime."
161+
}
162+
}
163+
164+
config.iterationTimeUnit?.let {
165+
val validTimeUnits = setOf("seconds", "s", "microseconds", "us", "milliseconds", "ms", "nanoseconds", "ns", "minutes", "m")
166+
require(it.toLowerCase() in validTimeUnits) {
167+
"Invalid iterationTimeUnit: '$it'. Accepted units: ${validTimeUnits.joinToString(", ")} (e.g., iterationTimeUnit = 'ms')."
168+
}
169+
require(config.iterationTime != null) {
170+
"Missing iterationTime. Please provide iterationTime when specifying iterationTimeUnit."
171+
}
172+
}
173+
174+
config.mode?.let {
175+
val validModes = setOf("thrpt", "avgt")
176+
require(it.toLowerCase() in validModes) {
177+
"Invalid benchmark mode: '$it'. Accepted modes: ${validModes.joinToString(", ")} (e.g., mode = 'thrpt')."
178+
}
179+
}
180+
181+
config.outputTimeUnit?.let {
182+
val validTimeUnits = setOf("seconds", "s", "microseconds", "us", "milliseconds", "ms", "nanoseconds", "ns", "minutes", "m")
183+
require(it.toLowerCase() in validTimeUnits) {
184+
"Invalid outputTimeUnit: '$it'. Accepted units: ${validTimeUnits.joinToString(", ")} (e.g., outputTimeUnit = 'ns')."
185+
}
186+
}
187+
188+
config.includes.forEach { pattern ->
189+
require(pattern.isNotBlank()) {
190+
"Invalid include pattern: '$pattern'. Pattern must not be blank."
191+
}
192+
}
193+
194+
config.excludes.forEach { pattern ->
195+
require(pattern.isNotBlank()) {
196+
"Invalid exclude pattern: '$pattern'. Pattern must not be blank."
197+
}
198+
}
199+
200+
config.params.forEach { (param, values) ->
201+
require(param.isNotBlank()) {
202+
"Invalid param name: '$param'. It must not be blank."
203+
}
204+
require(values.isNotEmpty()) {
205+
"Param '$param' has no values. At least one value is required."
206+
}
207+
}
208+
209+
config.advanced.forEach { (param, value) ->
210+
println("Validating advanced param: $param with value: $value")
211+
require(param.isNotBlank()) {
212+
"Invalid advanced config param: '$param'. It must not be blank."
213+
}
214+
require(value.toString().isNotBlank()) {
215+
"Invalid value for param '$param': '$value'. Value should not be blank."
216+
}
217+
218+
when (param) {
219+
"nativeFork" -> {
220+
val validValues = setOf("perbenchmark", "periteration")
221+
require(value.toString().toLowerCase() in validValues) {
222+
"Invalid value for 'nativeFork': '$value'. Accepted values: ${validValues.joinToString(", ")}."
223+
}
224+
}
225+
"nativeGCAfterIteration" -> require(value is Boolean) {
226+
"Invalid value for 'nativeGCAfterIteration': '$value'. Expected a Boolean value."
227+
}
228+
"jvmForks" -> {
229+
val intValue = value.toString().toIntOrNull()
230+
require(intValue != null && intValue >= 0 || value.toString().toLowerCase() == "definedbyjmh") {
231+
"Invalid value for 'jvmForks': '$value'. Expected a non-negative integer or 'definedByJmh'."
232+
}
233+
}
234+
"jsUseBridge" -> require(value is Boolean) {
235+
"Invalid value for 'jsUseBridge': '$value'. Expected a Boolean value."
236+
}
237+
else -> throw IllegalArgumentException("Invalid advanced config parameter: '$param'. Accepted parameters: 'nativeFork', 'nativeGCAfterIteration', 'jvmForks', 'jsUseBridge'.")
238+
}
239+
}
142240
}
143241

144242
internal val Gradle.isConfigurationCacheAvailable

0 commit comments

Comments
 (0)