Benchmarks setup [2]

Author: whyoleg

.github/workflows/ci-benchmarks.yml

@@ -3,7 +3,7 @@ on:
   push:
 
 jobs:
-  benchmark:
+  benchmark-local:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -15,22 +15,62 @@ jobs:
 
       - run: >
           ./gradlew
-          FastCsvLocalRequestChannelBenchmark
-          FastCsvLocalRequestResponseBenchmark
-          FastCsvLocalRequestStreamBenchmark
-          --dry-run
+          LocalRequestResponseBenchmark
+          LocalRequestStreamBenchmark
+          LocalRequestChannelBenchmark
+          -Prsocketbuild.skipBenchmarkTasks=true
+          --no-daemon
 
       - run: >
           ./gradlew
-          FastCsvLocalRequestChannelBenchmark
-          FastCsvLocalRequestResponseBenchmark
-          FastCsvLocalRequestStreamBenchmark
+          LocalRequestResponseBenchmark
+          LocalRequestStreamBenchmark
+          LocalRequestChannelBenchmark
           --no-parallel
           --max-workers=1
+          --no-daemon
+          --continue
 
       - if: always() && !cancelled()
         uses: actions/upload-artifact@v4
         with:
-          name: benchmark-reports-${{ matrix.os }}
-          path: "benchmarks/**/build/reports/benchmarks/**/*.csv"
-          retention-days: 1
+          name: benchmark-reports-local-${{ matrix.os }}
+          path: "rsocket-transport-benchmarks/**/build/reports/benchmarks/**/*.csv"
+          retention-days: 7
+
+  benchmark-network:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-gradle
+
+      - run: >
+          ./gradlew
+          KtorTcpRequestResponseBenchmark
+          KtorWsRequestResponseBenchmark
+          NettyTcpRequestResponseBenchmark
+          NettyQuicRequestResponseBenchmark
+          -Prsocketbuild.skipBenchmarkTasks=true
+          --no-daemon
+
+      - run: >
+          ./gradlew
+          KtorTcpRequestResponseBenchmark
+          KtorWsRequestResponseBenchmark
+          NettyTcpRequestResponseBenchmark
+          NettyQuicRequestResponseBenchmark
+          --no-parallel
+          --max-workers=1
+          --no-daemon
+          --continue
+
+      - if: always() && !cancelled()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-reports-network-${{ matrix.os }}
+          path: "rsocket-transport-benchmarks/**/build/reports/benchmarks/**/*.csv"
+          retention-days: 7

benchmarks/rsocket-java/src/jvmMain/kotlin/WsRSocketJavaBenchmark.kt

@@ -14,6 +14,10 @@
  * limitations under the License.
  */
 
+import gradle.kotlin.dsl.accessors._5e24a5abecdb427308a6c627b6f26b4a.allOpen
+import gradle.kotlin.dsl.accessors._5e24a5abecdb427308a6c627b6f26b4a.benchmark
+import kotlinx.benchmark.gradle.JvmBenchmarkTarget
+
 plugins {
     id("rsocketbuild.multiplatform-base")
     kotlin("plugin.allopen")
@@ -23,3 +27,16 @@ plugins {
 allOpen {
     annotation("org.openjdk.jmh.annotations.State")
 }
+
+benchmark {
+    targets.configureEach {
+        if (name == "jvm") {
+            this as JvmBenchmarkTarget
+            jmhVersion = "1.37"
+        }
+    }
+}
+
+if (providers.gradleProperty("rsocketbuild.skipBenchmarkTasks").map(String::toBoolean).getOrElse(false)) {
+    tasks.named { it.endsWith("Benchmark") }.configureEach { onlyIf { false } }
+}

build-logic/src/main/kotlin/rsocketbuild.multiplatform-benchmarks.gradle.kts

@@ -19,50 +19,25 @@ package rsocketbuild
 import kotlinx.benchmark.gradle.*
 import org.gradle.kotlin.dsl.support.*
 
-fun BenchmarksExtension.registerBenchmarks(
+fun BenchmarksExtension.registerTransportBenchmarks(
     implName: String,
     transports: List<String>,
     customize: BenchmarkConfiguration.(transport: String) -> Unit = {},
 ) {
-    fun register(
-        operation: String,
-        transport: String,
-        format: String,
-        kind: String,
-    ) {
-        configurations.register(
-            listOf(kind, format, transport, operation).joinToString("", transform = String::uppercaseFirstChar)
-        ) {
-            reportFormat = format
-            when (kind) {
-                "fast" -> param("payloadSize", "0")
-                "full" -> param("payloadSize", "0", "64", "4096")
-                else   -> error("wrong `kind`=$kind")
-            }
-
-            include("${transport.uppercaseFirstChar()}${implName}Benchmark.${operation}Blocking")
-            include("${transport.uppercaseFirstChar()}${implName}Benchmark.${operation}Parallel")
-            include("${transport.uppercaseFirstChar()}${implName}Benchmark.${operation}Concurrent")
-
-            customize(transport)
-        }
-    }
-
     val operations = listOf("requestResponse", "requestStream", "requestChannel")
-    val formats = listOf("text", "csv", "json")
-    val kinds = listOf("fast", "full")
     transports.forEach { transport ->
         operations.forEach { operation ->
-            formats.forEach { format ->
-                kinds.forEach { kind ->
-                    register(operation, transport, format, kind)
-                }
+            configurations.register(
+                listOf(transport, operation).joinToString("", transform = String::uppercaseFirstChar)
+            ) {
+                reportFormat = "csv"
+
+                include("${transport.uppercaseFirstChar()}${implName}Benchmark.${operation}Blocking")
+                include("${transport.uppercaseFirstChar()}${implName}Benchmark.${operation}Parallel")
+                include("${transport.uppercaseFirstChar()}${implName}Benchmark.${operation}Concurrent")
+
+                customize(transport)
             }
         }
     }
 }
-
-//if (transport == "local") {
-//    param("dispatcher", "DEFAULT", "UNCONFINED")
-////                        param("channels", "S", "M")
-//}

build-logic/src/main/kotlin/rsocketbuild/benchmarks.kt

@@ -17,10 +17,6 @@ bouncycastle = "1.80"
 
 turbine = "1.2.0"
 
-rsocket-java = "1.1.4"
-
-jmh = "1.37"
-
 maven-publish = "0.30.0"
 
 [libraries]
@@ -48,10 +44,6 @@ bouncycastle = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref = "boun
 
 turbine = { module = "app.cash.turbine:turbine", version.ref = "turbine" }
 
-rsocket-java-core = { module = 'io.rsocket:rsocket-core', version.ref = "rsocket-java" }
-rsocket-java-transport-local = { module = 'io.rsocket:rsocket-transport-local', version.ref = "rsocket-java" }
-rsocket-java-transport-netty = { module = 'io.rsocket:rsocket-transport-netty', version.ref = "rsocket-java" }
-
 kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" }
 kotlin-allopen-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-allopen", version.ref = "kotlin" }
 kotlinx-atomicfu-gradle-plugin = { module = "org.jetbrains.kotlinx:atomicfu-gradle-plugin", version.ref = "kotlinx-atomicfu" }

gradle/libs.versions.toml

@@ -14,18 +14,18 @@
  * limitations under the License.
  */
 
-package io.rsocket.kotlin.benchmarks
+package io.rsocket.kotlin.transport.benchmarks
 
 import kotlinx.coroutines.*
 import kotlinx.coroutines.flow.*
 
-const val ITERATION = 5
+const val ITERATION = 3
 const val ITERATION_DURATION = 3
 
-const val WARMUP = 5
+const val WARMUP = 3
 const val WARMUP_DURATION = 3
 
-abstract class RSocketBenchmark<Payload : Any, Blackhole : Any> {
+abstract class RSocketTransportBenchmark<Payload : Any, Blackhole : Any> {
 
     // payload operations
     abstract val payloadSize: Int
@@ -45,7 +45,6 @@ abstract class RSocketBenchmark<Payload : Any, Blackhole : Any> {
 
     // benchmarks
 
-    // Benchmark annotation doesn't inherit on jvm
     open fun requestResponseBlocking(bh: Blackhole) = blocking(bh, 1000, ::requestResponse)
     open fun requestResponseParallel(bh: Blackhole) = parallel(bh, 1000, ::requestResponse)
     open fun requestResponseConcurrent(bh: Blackhole) = concurrent(bh, 1000, ::requestResponse)

benchmarks/shared/build.gradle.kts renamed to rsocket-transport-benchmarks/base/build.gradle.kts

@@ -26,22 +26,18 @@ kotlin {
 
     sourceSets {
         jvmMain.dependencies {
-            implementation(projects.benchmarksShared)
-
+            implementation(projects.rsocketTransportBenchmarksBase)
             implementation(libs.kotlinx.coroutines.reactor)
-            implementation(libs.rsocket.java.transport.local)
-            implementation(libs.rsocket.java.transport.netty)
+
+            implementation("io.rsocket:rsocket-transport-local:1.1.5")
+            implementation("io.rsocket:rsocket-transport-netty:1.1.5")
         }
     }
 }
 
 benchmark {
     targets {
-        register("jvm") {
-            this as JvmBenchmarkTarget
-            jmhVersion = libs.versions.jmh.get()
-        }
+        register("jvm")
     }
-
-    registerBenchmarks("RSocketJava", listOf("local", "tcp", "ws"))
+    registerTransportBenchmarks("RSocketJava", listOf("local", "nettyTcp"))
 }

benchmarks/shared/src/commonMain/kotlin/RSocketBenchmark.kt renamed to rsocket-transport-benchmarks/base/src/commonMain/kotlin/RSocketTransportBenchmark.kt

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.rsocket.kotlin.benchmarks.java
+package io.rsocket.kotlin.transport.benchmarks.java
 
 import io.rsocket.transport.*
 import io.rsocket.transport.local.*

benchmarks/rsocket-java/build.gradle.kts renamed to rsocket-transport-benchmarks/rsocket-java/build.gradle.kts

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-package io.rsocket.kotlin.benchmarks.java
+package io.rsocket.kotlin.transport.benchmarks.java
 
 import io.rsocket.transport.*
 import io.rsocket.transport.netty.client.*
 import io.rsocket.transport.netty.server.*
 
-class TcpRSocketJavaBenchmark : RSocketJavaBenchmark() {
+class NettyTcpRSocketJavaBenchmark : RSocketJavaBenchmark() {
     override val serverTransport: ServerTransport<*> = TcpServerTransport.create(9000)
     override val clientTransport: ClientTransport = TcpClientTransport.create(9000)
 }