Overhaul benchmarking infrastructure (#63)

Significantly reduces the overhead of benchmarking and provides more reliable real-world numbers

Author: PaintNinja

build.gradle

@@ -5,7 +5,7 @@ plugins {
     id 'java-library'
     id 'maven-publish'
     id 'com.github.ben-manes.versions' version '0.50.0'
-    id 'org.gradlex.extra-java-module-info' version '1.4.2'
+    id 'org.gradlex.extra-java-module-info' version '1.8'
     id 'net.minecraftforge.gradleutils' version '2.3.3'
     id 'net.minecraftforge.licenser' version '1.0.1'
 }
@@ -16,7 +16,7 @@ print("Version: $version")
 
 java {
     toolchain.languageVersion = JavaLanguageVersion.of(16)
-    modularity.inferModulePath.set(true)
+    modularity.inferModulePath = true
     withSourcesJar()
 }
 
@@ -101,4 +101,4 @@ allprojects {
     // Tests are expensive to run all variants, so only run if asked to
     if (!project.hasProperty('bulk_tests'))
         ext.VALID_VMS = ['Adoptium':  [17]]//, 18, 19, 20, 21] ]
-}
+}

buildSrc/src/main/groovy/AggregateJmh.groovy

@@ -38,7 +38,7 @@ abstract class AggregateJmh extends DefaultTask {
 
         int javaCount = 0
         for (def file : this.inputs.files) {
-            if (file.equals(pastFile))
+            if (file == pastFile)
                 continue
             def (javaName,javaVersion) = file.name.substring('jmh-'.length(), file.name.length() - 5).split('-')
 
@@ -61,8 +61,8 @@ abstract class AggregateJmh extends DefaultTask {
             }
         }
 
-        def onlyOneSuite = results.keySet().size() == 1 && results.values().iterator().next().keySet().size() == 1
-        def loadOld = collate.isPresent() && collate.get()
+        boolean onlyOneSuite = results.keySet().size() == 1 && results.values().iterator().next().keySet().size() == 1
+        boolean loadOld = collate.isPresent() && collate.get()
 
         def markdown,csv = 'Something went fucky'
         if (javaCount == 1) {
@@ -85,7 +85,7 @@ abstract class AggregateJmh extends DefaultTask {
         outputCsv.asFile.get().text = csv
     }
 
-    static def rsplit(def str, def del, int limit = -1) {
+    static def rsplit(def str, String del, int limit = -1) {
         def lst = []
         def x = 0, idx
         def tmp = str
@@ -97,7 +97,7 @@ abstract class AggregateJmh extends DefaultTask {
         return lst
     }
 
-    static def mergePast(def to, def from) {
+    static void mergePast(def to, def from) {
         def pkg = to.keySet().iterator().next()
         if (!from.containsKey(pkg))
             return
@@ -135,7 +135,7 @@ abstract class AggregateJmh extends DefaultTask {
      *
      * This allows for comparison between multiple environments and implementations
      */
-    static def formatCollated(def results, def resultsPast, def timeUnit) {
+    static def formatCollated(def results, def resultsPast, String timeUnit) {
         def columns = [' ']
         def rows = []
         def base = [:]
@@ -213,7 +213,7 @@ abstract class AggregateJmh extends DefaultTask {
         return markdown
     }
 
-    static def buildCsv(def columns, def rows, def headers = true, def prefix = '') {
+    static String buildCsv(def columns, def rows, boolean headers = true, String prefix = '') {
         columns = [] + columns
         rows = [] + rows
         for (int x = 0; x < rows.size(); x++)
@@ -226,7 +226,7 @@ abstract class AggregateJmh extends DefaultTask {
                 x--
             }
         }
-        def csv = ''
+        String csv = ''
         if (headers)
             csv += columns.join('\t') + '\n'
         for (def row : rows)
@@ -252,9 +252,9 @@ abstract class AggregateJmh extends DefaultTask {
      * | postStaticHundred  |   5878.851 |   5325.055 |      -9% |
      * Relative - Percentage change vs first suite
      */
-    static def formatBulk(def results, def resultsPast, def timeUnit, def javas, def versions) {
-        def markdown = ''
-        def onlyOneSuite = results.keySet().size() == 1 && results.values().iterator().next().keySet().size() == 1
+    static def formatBulk(def results, def resultsPast, String timeUnit, def javas, def versions) {
+        String markdown = ''
+        boolean onlyOneSuite = results.keySet().size() == 1 && results.values().iterator().next().keySet().size() == 1
         def columns = [' ']
         def csvTable = [:]
         versions.forEach { version ->  columns += [version, 'Change', 'Relative'] }
@@ -330,7 +330,7 @@ abstract class AggregateJmh extends DefaultTask {
                 }
             }
         }
-        def csvData = ''
+        String csvData = ''
         for (def line : csv)
             csvData += line.join('\t') + '\n'
 

eventbus-jmh/build.gradle

@@ -28,11 +28,11 @@ dependencies {
     implementation(libs.securemodules)
     implementation(libs.modlauncher)
     implementation(libs.unsafe)
-    implementation(project(':eventbus-test-jar'))
+    implementation(projects.eventbusTestJar)
 
-    implementation('org.openjdk.jmh:jmh-core:1.37')
-    runtimeOnly('org.openjdk.jmh:jmh-generator-annprocess:1.37')
-    annotationProcessor('org.openjdk.jmh:jmh-generator-annprocess:1.37')
+    implementation libs.jmh.core
+    runtimeOnly libs.jmh.annotationProcessor
+    annotationProcessor libs.jmh.annotationProcessor
 }
 
 extraJavaModuleInfo {
@@ -57,15 +57,16 @@ tasks.register('archiveJfr', ArchiveJfr) {
     output = file('build/jmh_profile_results/archive/')
 }
 
-tasks.register('jmh').configure {
-    dependsOn('aggregateJmh', 'archiveJfr')
+tasks.register('jmh') {
+    dependsOn 'aggregateJmh'
 }
 
-VALID_VMS.each { javaVendor, javaVersions ->
-    javaVersions.each { int javaVersion ->
-        def output = file("build/jmh_results/jmh-${javaVendor}-${javaVersion}.json")
-        def outputJfr = file("build/jmh_profile_results/last_run/${javaVendor}-${javaVersion}/")
-        def task = tasks.register("jmh${javaVendor}${javaVersion}", JavaExec) {
+((Map<String, List<Integer>>) ext.VALID_VMS).forEach { javaVendor, javaVersions ->
+    for (Integer javaVersion in javaVersions) {
+        var output = file("build/jmh_results/jmh-${javaVendor}-${javaVersion}.json")
+        var outputJfr = file("build/jmh_profile_results/last_run/${javaVendor}-${javaVersion}/")
+        outputJfr.mkdirs()
+        var task = tasks.register("jmh${javaVendor}${javaVersion}", JavaExec) {
             classpath = sourceSets.main.runtimeClasspath
             mainModule = 'net.minecraftforge.eventbus.jmh'
             mainClass = 'net.minecraftforge.eventbus.benchmarks.Main'
@@ -81,32 +82,44 @@ VALID_VMS.each { javaVendor, javaVersions ->
                     '-tu', 'ns',  // Time unit: [m, s, ms, us, ns]
                     '-f',  '1',   // Forks per benchmark
                     '-rf', 'json', // Results File Format
-                    '-prof', 'stack', // Profiler: Simple and naive Java stack profiler
-                    '-prof', "jfr:dir=${outputJfr}", // Profiler: Java Flight Recorder profiler
+
+                    // Todo: Conditionally enable these when given a debug flag. Log a warning if they are enabled
+                    //       saying that they should only be used for identifying hotspots and to not trust the
+                    //       results of the benchmarks when they are enabled.
+//                    '-prof', 'stack', // Profiler: Simple and naive Java stack profiler
+//                    '-prof', "jfr:dir=${outputJfr}", // Profiler: Java Flight Recorder profiler
+
+                    // Comment out the line below to enable BenchmarkClassLoader. This sets up a transformed environment
+                    // with ModLauncher but *doesn't* call useModLauncher() in BusBuilder.
+                    '-e', 'BenchmarkClassLoader.*',
+
                     '-v', 'EXTRA', // Verbosity [SILENT, NORMAL, EXTRA]
                     '-foe', 'true', // Fail on error
                     '-rff', output,
             ]
             if (project.hasProperty('bench'))
                 args += project.property('bench')
-                
+
             doFirst {
-                if (outputJfr.exists())
-                    outputJfr.deleteDir()
+                outputJfr.deleteDir()
                 if (!output.parentFile.exists())
-                    output.parentFile.mkdirs() 
+                    output.parentFile.mkdirs()
                 if (output.exists())
                     output.delete()
             }
-            javaLauncher.set(javaToolchains.launcherFor {
-                it.vendor.set(JvmVendorSpec."${javaVendor.toUpperCase(Locale.ROOT)}" as JvmVendorSpec)
-                it.languageVersion.set(JavaLanguageVersion.of(javaVersion))
-                it.implementation.set(JvmImplementation.VENDOR_SPECIFIC)
-            })
+            javaLauncher = javaToolchains.launcherFor {
+                vendor = JvmVendorSpec."${javaVendor.toUpperCase(Locale.ROOT)}" as JvmVendorSpec
+                languageVersion = JavaLanguageVersion.of(javaVersion)
+                implementation = JvmImplementation.VENDOR_SPECIFIC
+            }
+        }
+        tasks.named('aggregateJmh') {
+            dependsOn task
+            inputs.file output
+        }
+        tasks.named('archiveJfr') {
+            dependsOn task
+            inputs.dir outputJfr
         }
-        aggregateJmh.dependsOn(task)
-        aggregateJmh.inputs.file(output)
-        archiveJfr.dependsOn(task)
-        archiveJfr.inputs.dir(outputJfr)
     }
 }