Updated default JMH version to 1.37 and added its validation (#312)

If a project explicitly uses an older JMH version,
the plugin will warn about it.
If there are multiple JVM targets configured with
different JMH versions,
the plugin will warn about an unsupported setup.

Closes #147

Author: fzhinkin

docs/configuration-options.md

@@ -84,21 +84,29 @@ benchmark {
 - **"definedByJmh"** – Let JMH determine the amount, using the value in the [`@Fork` annotation](https://javadoc.io/doc/org.openjdk.jmh/jmh-core/latest/org/openjdk/jmh/annotations/Fork.html) for the benchmark function or its enclosing class. If not specified by `@Fork`, it defaults to [Defaults.MEASUREMENT_FORKS (`5`)](https://javadoc.io/doc/org.openjdk.jmh/jmh-core/latest/org/openjdk/jmh/runner/Defaults.html#MEASUREMENT_FORKS).
 
 The library offers the flexibility to specify the version of the Java Microbenchmark Harness (JMH) to use when running benchmarks on the JVM.
-The default version is set to `1.21`, but you can customize it while registering a JVM target for benchmarking:
+The default version is set to `1.37`, but you can customize it while registering a JVM target for benchmarking:
 
 ```kotlin
 benchmark {
     targets {
         register("jvmBenchmarks") {
             this as JvmBenchmarkTarget
-            jmhVersion = "1.36"
+            jmhVersion = "1.38"
         }
     }
 }
 ```
 
 Alternatively, you can utilize the project property `benchmarks_jmh_version` to achieve the same effect.
 
+> [!WARNING]
+> While it is possible to register multiple JVM benchmark targets with different JMH versions,
+> such configurations are not supported. Using such configurations may result in runtime errors.
+
+> [!NOTE]
+> It is recommended to change JMH version only when a new JMH version was released,
+> but `kotlinx-benchmark` plugin applied to a project is still using an older version.
+
 ### Kotlin/JS & Kotlin/Wasm
 | Option                                        | Description                                                                                           | Possible Values | Default Value |
 |-----------------------------------------------|-------------------------------------------------------------------------------------------------------|-----------------|---------------|

examples/java/build.gradle.kts

@@ -25,7 +25,7 @@ benchmark {
     targets {
         register("main") {
             this as JvmBenchmarkTarget
-            jmhVersion = "1.21"
+            jmhVersion = "1.37"
         }
     }
 }

examples/kotlin-jvm-separate-benchmark-source-set/build.gradle.kts

@@ -43,7 +43,7 @@ benchmark {
     targets {
         register("benchmarks") {
             if (this is JvmBenchmarkTarget) {
-                jmhVersion = "1.21"
+                jmhVersion = "1.37"
             }
         }
     }

examples/kotlin-jvm/build.gradle.kts

@@ -36,9 +36,6 @@ benchmark {
         }
     }
     targets {
-        register("main") {
-            this as JvmBenchmarkTarget
-            jmhVersion = "1.21"
-        }
+        register("main")
     }
-}
+}

examples/kotlin-multiplatform/build.gradle

@@ -103,12 +103,12 @@ benchmark {
         // This one matches target name, e.g. 'jvm', 'js',
         // and registers its 'main' compilation, so 'jvm' registers 'jvmMain'
         register("jvm") {
-            jmhVersion = "1.21"
+            jmhVersion = "1.37"
         }
         // This one matches source set name, e.g. 'jvmMain', 'jvmTest', etc
         // and register the corresponding compilation (here the 'benchmark' compilation declared in the 'jvm' target)
         register("jvmBenchmark") {
-            jmhVersion = "1.21"
+            jmhVersion = "1.37"
         }
         register("jsDefaultExecutor")
         register("jsBuiltInExecutor") {

gradle/libs.versions.toml

@@ -6,7 +6,7 @@ kotlin-for-gradle-plugin = "2.0.20" # Kotlin 2.1 removes support for the used la
 kotlinx-binaryCompatibilityValidator = "0.16.2"
 kotlinx-teamInfra = "0.4.0-dev-85"
 squareup-kotlinpoet = "1.3.0"
-jmh = "1.21"
+jmh = "1.37"
 gradle-pluginPublish = "0.21.0"
 
 # Note: This version can be overridden by passing `-Pmin_supported_gradle_version=<version>`

integration/src/test/kotlin/kotlinx/benchmark/integration/JmhVersionValidationTest.kt

@@ -0,0 +1,19 @@
+package kotlinx.benchmark.integration
+
+import kotlin.test.Test
+
+class JmhVersionValidationTest : GradleTest() {
+    @Test
+    fun verifyWarningsAboutJmhVersions() {
+        val runner = project("conflicting-jmh-versions", true) {}
+
+        runner.runAndSucceed("assembleBenchmarks") {
+            assertOutputContains("configures several JVM benchmarking targets that use different " +
+                    "JMH versions (1.21 is used by jvmFirst; 1.22 is used by jvmSecond, jvmThird). " +
+                    "Such configuration is not supported and may lead to runtime errors. " +
+                    "Consider using the same JMH version across all benchmarking targets.")
+            assertOutputContains("Configured JMH version (1.22) is older than a default version supplied by the benchmarking plugin")
+            assertOutputContains("Configured JMH version (1.21) is older than a default version supplied by the benchmarking plugin")
+        }
+    }
+}

integration/src/test/resources/templates/conflicting-jmh-versions/build.gradle

@@ -0,0 +1,15 @@
+kotlin {
+    jvm {
+        compilations.create('first') { associateWith(compilations.main) }
+        compilations.create('second') { associateWith(compilations.main) }
+        compilations.create('third') { associateWith(compilations.main) }
+    }
+}
+
+benchmark {
+    targets {
+        register("jvmFirst") { jmhVersion = "1.21" }
+        register("jvmSecond") { jmhVersion = "1.22" }
+        register("jvmThird") { jmhVersion = "1.22" }
+    }
+}

integration/src/test/resources/templates/conflicting-jmh-versions/gradle.properties

@@ -0,0 +1 @@
+org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8

plugin/build.gradle.kts

@@ -141,6 +141,9 @@ val generatePluginConstants by tasks.registering {
     val kotlinCompilerVersion = libs.versions.kotlin.asProvider()
     inputs.property("kotlinCompilerVersion", kotlinCompilerVersion)
 
+    val defaultJvmVersion = libs.versions.jmh
+    inputs.property("defaultJmhVersion", defaultJvmVersion)
+
     doLast {
         constantsKtFile.writeText(
                 """|package kotlinx.benchmark.gradle.internal
@@ -150,6 +153,7 @@ val generatePluginConstants by tasks.registering {
                 |  const val MIN_SUPPORTED_GRADLE_VERSION = "${minSupportedGradleVersion.get()}"
                 |  const val MIN_SUPPORTED_KOTLIN_VERSION = "${minSupportedKotlinVersion.get()}"
                 |  const val DEFAULT_KOTLIN_COMPILER_VERSION = "${kotlinCompilerVersion.get()}"
+                |  const val DEFAULT_JMH_VERSION = "${defaultJvmVersion.get()}"
                 |}
                 |""".trimMargin()
         )