Improve error messages for invalid use of annotations

Author: Abduqodiri Qurbonzoda

docs/writing-benchmarks.md

@@ -87,6 +87,8 @@ The key point to remember is that the `@Setup` method's execution time is not in
 results - the timer starts only when the `@Benchmark` method begins. This makes `@Setup` an ideal place
 for initialization tasks that should not impact the timing results of your benchmark.
 
+The method annotated with `@Setup` should be `public` and have no arguments.
+In Kotlin/JVM, these restrictions are slightly less strict.
 Refer to [JMH documentation of @Setup](https://javadoc.io/doc/org.openjdk.jmh/jmh-core/latest/org/openjdk/jmh/annotations/Setup.html)
 for details about the effect and restrictions of the annotation in Kotlin/JVM.
 
@@ -106,6 +108,8 @@ The `@TearDown` annotation is crucial for avoiding performance bias, ensuring th
 and preparing a clean environment for the next run. Similar to the `@Setup` method, the execution time of the
 `@TearDown` method is not included in the final benchmark results.
 
+The method annotated with `@TearDown` should be `public` and have no arguments.
+In Kotlin/JVM, these restrictions are slightly less strict.
 Refer to [JMH documentation of @TearDown](https://javadoc.io/doc/org.openjdk.jmh/jmh-core/latest/org/openjdk/jmh/annotations/TearDown.html)
 for more information on the effect and restrictions of the annotation in Kotlin/JVM.
 
@@ -118,7 +122,7 @@ It's the actual test you're running. The code you want to benchmark goes inside
 All other annotations are employed to configure the benchmark's environment and execution.
 
 Benchmark methods may include only a single [Blackhole](#blackhole) type as an argument, or have no arguments at all.
-It's important to note that in Kotlin/JVM benchmark methods must always be `public`.
+It's important to note that benchmark methods should be `public`. In Kotlin/JVM, these restrictions are slightly less strict. 
 Refer to [JMH documentation of @Benchmark](https://javadoc.io/doc/org.openjdk.jmh/jmh-core/latest/org/openjdk/jmh/annotations/Benchmark.html)
 for details about restrictions for benchmark methods in Kotlin/JVM.
 
@@ -194,11 +198,15 @@ with each iteration lasting one second, for the final performance measurement.
 
 The `@Param` annotation is used to pass different parameters to your benchmark method.
 It allows you to run the same benchmark method with different input values, so you can see how these variations affect
-performance. The values you provide for the `@Param` annotation are the different inputs you want to use in your
-benchmark test. The benchmark will run once for each provided value.
+performance.
 
-The property marked with this annotation must be mutable (`var`) and not `private.`
+The values provided by the `@Param` annotation represent the different inputs you want to use in your benchmark.
+Since the benchmark runs once for each provided value, the annotation should have at least one argument.
+The annotation values are given in `String` and will be coerced as needed to match the property type.
+
+The property marked with this annotation should be mutable (`var`) and `public`.
 Additionally, only properties of primitive types or the `String` type can be annotated with `@Param`.
+In Kotlin/JVM, these restrictions are slightly less strict.
 Refer to [JMH documentation of @Param](https://javadoc.io/doc/org.openjdk.jmh/jmh-core/latest/org/openjdk/jmh/annotations/Param.html)
 for details about the effect and restrictions of the annotation in Kotlin/JVM.
 

integration/src/main/kotlin/kotlinx/benchmark/integration/AnnotationsSpecifier.kt

@@ -1,26 +1,111 @@
 package kotlinx.benchmark.integration
 
+@OptIn(ExperimentalStdlibApi::class)
 class AnnotationsSpecifier {
-    private var isMeasurementSpecified: Boolean = false
-    private var iterations: Int? = null
-    private var time: Int? = null
-    private var timeUnit: String? = null
+    private val classAnnotations = mutableListOf<Annotation>()
+    private val propertyAnnotations = mutableListOf<AnnotatedMember>()
+    private val functionAnnotations = mutableListOf<AnnotatedMember>()
 
     fun measurement(iterations: Int, time: Int, timeUnit: String) {
-        isMeasurementSpecified = true
-        this.iterations = iterations
-        this.time = time
-        this.timeUnit = timeUnit
+        classAnnotations.add(
+            Annotation("@Measurement", listOf(iterations, time, timeUnit))
+        )
     }
 
-    fun replacementForLine(line: String): String {
+    fun outputTimeUnit(timeUnit: String) {
+        classAnnotations.add(
+            Annotation("@OutputTimeUnit", listOf(timeUnit))
+        )
+    }
+
+    fun benchmarkMode(mode: String) {
+        classAnnotations.add(
+            Annotation("@BenchmarkMode", listOf(mode))
+        )
+    }
+
+    fun benchmark(functionName: String) {
+        functionAnnotations.add(
+            AnnotatedMember(functionName, Annotation("@Benchmark"))
+        )
+    }
+
+    fun setup(functionName: String) {
+        functionAnnotations.add(
+            AnnotatedMember(functionName, Annotation("@Setup"))
+        )
+    }
+
+    fun teardown(functionName: String) {
+        functionAnnotations.add(
+            AnnotatedMember(functionName, Annotation("@TearDown"))
+        )
+    }
+
+    fun param(propertyName: String, vararg values: String) {
+        require(values.all { '\"' !in it }) { "TODO: Support param values that contain '\"'." }
+
+        propertyAnnotations.add(
+            AnnotatedMember(propertyName, Annotation("@Param", values.map { "\"$it\"" }))
+        )
+    }
+
+    fun annotationsForProperty(line: String): List<String> {
+        val annotations = mutableListOf<String>()
+        for ((propertyName, annotation) in propertyAnnotations) {
+            val regex = Regex("\\s*(public|private|protected|internal)?\\s*(final|open)?\\s*(val|var)\\s+${Regex.escape(propertyName)}")
+            if (regex.matchesAt(line, 0)) {
+                check(!annotation.isUsed)
+                annotation.isUsed = true
+                annotations.add(annotation.toCode())
+            }
+        }
+        return annotations
+    }
+    
+    fun annotationsForFunction(line: String): List<String> {
+        val annotations = mutableListOf<String>()
+        for ((functionName, annotation) in functionAnnotations) {
+            val regex = Regex("\\s*(public|private|protected|internal)?\\s*(final|open)?\\s*fun\\s+${Regex.escape(functionName)}\\(")
+            if (regex.matchesAt(line, 0)) {
+                check(!annotation.isUsed)
+                annotation.isUsed = true
+                annotations.add(annotation.toCode())
+            }
+        }
+        return annotations
+    }
+
+    fun replaceClassAnnotation(line: String): String {
         val trimmedLine = line.trimStart()
         val prefix = line.substring(0, line.length - trimmedLine.length)
-        return when {
-            isMeasurementSpecified && trimmedLine.startsWith("@Measurement") ->
-                "$prefix@Measurement($iterations, $time, $timeUnit)"
-            else ->
-                line
+        for (annotation in classAnnotations) {
+            if (trimmedLine.startsWith(annotation.name)) {
+                check(!annotation.isUsed)
+                annotation.isUsed = true
+                return prefix + annotation.toCode()
+            }
         }
+        return line
+    }
+
+    fun checkAllAnnotationsAreUsed() {
+        classAnnotations.forEach { check(it.isUsed) { "Unused class annotation: $it" } }
+        propertyAnnotations.forEach { check(it.annotation.isUsed) { "Unused property annotation: $it" } }
+        functionAnnotations.forEach { check(it.annotation.isUsed) { "Unused function annotation: $it" } }
     }
+}
+
+private data class AnnotatedMember(
+    val memberName: String,
+    val annotation: Annotation
+)
+
+private data class Annotation(
+    val name: String,
+    val arguments: List<Any?> = emptyList(),
+    var isUsed: Boolean = false
+) {
+    fun toCode(): String =
+        "$name${if (arguments.isEmpty()) "" else arguments.joinToString(", ", "(", ")")}"
 }

integration/src/main/kotlin/kotlinx/benchmark/integration/Runner.kt

@@ -36,8 +36,40 @@ class Runner(
     fun updateAnnotations(filePath: String, annotationsSpecifier: AnnotationsSpecifier.() -> Unit) {
         val annotations = AnnotationsSpecifier().also(annotationsSpecifier)
         val file = projectDir.resolve(filePath)
-        val updatedLines = file.readLines().map { annotations.replacementForLine(it) }
+
+        val updatedLines = file.readLines().map {
+            annotations.replaceClassAnnotation(it)
+        }
+        annotations.checkAllAnnotationsAreUsed()
+
+        file.writeText(updatedLines.joinToString(separator = "\n"))
+        if (print) {
+            println(file.readText())
+        }
+    }
+
+    fun addAnnotation(filePath: String, annotationsSpecifier: AnnotationsSpecifier.() -> Unit) {
+        val annotations = AnnotationsSpecifier().also(annotationsSpecifier)
+        val file = projectDir.resolve(filePath)
+
+        val updatedLines = mutableListOf<String>()
+
+        file.readLines().forEach { line ->
+            val indentation = " ".repeat(line.length - line.trimStart().length)
+            annotations.annotationsForFunction(line).forEach { annotation ->
+                updatedLines.add(indentation + annotation)
+            }
+            annotations.annotationsForProperty(line).forEach { annotation ->
+                updatedLines.add(indentation + annotation)
+            }
+            updatedLines.add(line)
+        }
+        annotations.checkAllAnnotationsAreUsed()
+
         file.writeText(updatedLines.joinToString(separator = "\n"))
+        if (print) {
+            println(file.readText())
+        }
     }
 
     fun generatedDir(targetName: String, filePath: String, fileTestAction: (File) -> Unit) {