Setup benchmarks with CI

Author: whyoleg

.github/workflows/ci-benchmarks.yml

@@ -0,0 +1,37 @@
+name: Benchmarks CI
+on:
+  push:
+
+jobs:
+  benchmark:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-gradle
+      - run: ./gradlew assembleBenchmarks
+
+      - run: >
+          ./gradlew
+          FastCsvLocalRequestChannelBenchmark
+          FastCsvLocalRequestResponseBenchmark
+          FastCsvLocalRequestStreamBenchmark
+          --dry-run
+
+      - run: >
+          ./gradlew
+          FastCsvLocalRequestChannelBenchmark
+          FastCsvLocalRequestResponseBenchmark
+          FastCsvLocalRequestStreamBenchmark
+          --no-parallel
+          --max-workers=1
+
+      - if: always() && !cancelled()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-reports-${{ matrix.os }}
+          path: "benchmarks/**/build/reports/benchmarks/**/*.csv"
+          retention-days: 1

.github/workflows/ci-samples.yml

@@ -19,7 +19,5 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-gradle
-        with:
-          cache-read-only: true
       - run: ./gradlew build --continue
         working-directory: samples/${{ matrix.sample }}

benchmarks/rsocket-java/build.gradle.kts

@@ -14,12 +14,11 @@
  * limitations under the License.
  */
 
+import kotlinx.benchmark.gradle.*
 import rsocketbuild.*
 
 plugins {
-    id("rsocketbuild.multiplatform-base")
-    alias(libs.plugins.kotlin.plugin.allopen)
-    alias(libs.plugins.kotlinx.benchmark)
+    id("rsocketbuild.multiplatform-benchmarks")
 }
 
 kotlin {
@@ -36,32 +35,13 @@ kotlin {
     }
 }
 
-allOpen {
-    annotation("org.openjdk.jmh.annotations.State")
-}
-
 benchmark {
     targets {
         register("jvm") {
-            this as kotlinx.benchmark.gradle.JvmBenchmarkTarget
+            this as JvmBenchmarkTarget
             jmhVersion = libs.versions.jmh.get()
         }
     }
-    configurations {
-        configureEach {
-            reportFormat = "text"
-        }
-        named("main") {
-            // all params
-            param("payloadSize", "0", "64", "1024", "131072", "1048576")
-        }
-        register("localPayloadSize") {
-            include("LocalRSocketJavaBenchmark")
-            param("payloadSize", "0")
-        }
-        register("tcpPayloadSize") {
-            include("TcpRSocketJavaBenchmark")
-            param("payloadSize", "0", "64", "4096")
-        }
-    }
+
+    registerBenchmarks("RSocketJava", listOf("local", "tcp", "ws"))
 }

benchmarks/rsocket-java/src/jvmMain/kotlin/LocalRSocketJavaBenchmark.kt

@@ -18,18 +18,8 @@ package io.rsocket.kotlin.benchmarks.java
 
 import io.rsocket.transport.*
 import io.rsocket.transport.local.*
-import kotlinx.benchmark.*
-import kotlinx.coroutines.*
-import kotlinx.coroutines.channels.*
 
-@BenchmarkMode(Mode.Throughput)
-@Warmup(iterations = 3, time = 1)
-@Measurement(iterations = 3, time = 1)
-@State(Scope.Benchmark)
 class LocalRSocketJavaBenchmark : RSocketJavaBenchmark() {
-    @Param("0")
-    override var payloadSize: Int = 0
-
     override val serverTransport: ServerTransport<*> = LocalServerTransport.create("local")
     override val clientTransport: ClientTransport = LocalClientTransport.create("local")
 }

benchmarks/rsocket-java/src/jvmMain/kotlin/RSocketJavaBenchmark.kt

@@ -29,6 +29,10 @@ import org.reactivestreams.*
 import reactor.core.publisher.*
 import kotlin.random.*
 
+@BenchmarkMode(Mode.Throughput)
+@Warmup(iterations = WARMUP, time = WARMUP_DURATION)
+@Measurement(iterations = ITERATION, time = ITERATION_DURATION)
+@State(Scope.Benchmark)
 abstract class RSocketJavaBenchmark : RSocketBenchmark<Payload, Blackhole>() {
     protected abstract val clientTransport: ClientTransport
     protected abstract val serverTransport: ServerTransport<*>
@@ -96,27 +100,33 @@ abstract class RSocketJavaBenchmark : RSocketBenchmark<Payload, Blackhole>() {
         server.dispose()
     }
 
-//    @Benchmark
-//    override fun requestResponseBlocking(bh: Blackhole) = super.requestResponseBlocking(bh)
-//
-//    @Benchmark
-//    override fun requestResponseParallel(bh: Blackhole) = super.requestResponseParallel(bh)
-//
-//    @Benchmark
-//    override fun requestResponseConcurrent(bh: Blackhole) = super.requestResponseConcurrent(bh)
+    @Param("0")
+    override var payloadSize: Int = 0
 
     @Benchmark
-    override fun requestStreamBlocking(bh: Blackhole) {
-        super.requestStreamBlocking(bh)
-    }
+    override fun requestResponseBlocking(bh: Blackhole) = super.requestResponseBlocking(bh)
 
     @Benchmark
-    override fun requestStreamParallel(bh: Blackhole) {
-        super.requestStreamParallel(bh)
-    }
+    override fun requestResponseParallel(bh: Blackhole) = super.requestResponseParallel(bh)
 
     @Benchmark
-    override fun requestStreamConcurrent(bh: Blackhole) {
-        super.requestStreamConcurrent(bh)
-    }
+    override fun requestResponseConcurrent(bh: Blackhole) = super.requestResponseConcurrent(bh)
+
+    @Benchmark
+    override fun requestStreamBlocking(bh: Blackhole) = super.requestStreamBlocking(bh)
+
+    @Benchmark
+    override fun requestStreamParallel(bh: Blackhole) = super.requestStreamParallel(bh)
+
+    @Benchmark
+    override fun requestStreamConcurrent(bh: Blackhole) = super.requestStreamConcurrent(bh)
+
+    @Benchmark
+    override fun requestChannelBlocking(bh: Blackhole) = super.requestChannelBlocking(bh)
+
+    @Benchmark
+    override fun requestChannelParallel(bh: Blackhole) = super.requestChannelParallel(bh)
+
+    @Benchmark
+    override fun requestChannelConcurrent(bh: Blackhole) = super.requestChannelConcurrent(bh)
 }

benchmarks/rsocket-java/src/jvmMain/kotlin/TcpRSocketJavaBenchmark.kt

@@ -17,21 +17,10 @@
 package io.rsocket.kotlin.benchmarks.java
 
 import io.rsocket.transport.*
-import io.rsocket.transport.local.*
 import io.rsocket.transport.netty.client.*
 import io.rsocket.transport.netty.server.*
-import kotlinx.benchmark.*
-import kotlinx.coroutines.*
-import kotlinx.coroutines.channels.*
 
-@BenchmarkMode(Mode.Throughput)
-@Warmup(iterations = 3, time = 1)
-@Measurement(iterations = 3, time = 1)
-@State(Scope.Benchmark)
 class TcpRSocketJavaBenchmark : RSocketJavaBenchmark() {
-    @Param("0")
-    override var payloadSize: Int = 0
-
     override val serverTransport: ServerTransport<*> = TcpServerTransport.create(9000)
     override val clientTransport: ClientTransport = TcpClientTransport.create(9000)
 }

benchmarks/rsocket-java/src/jvmMain/kotlin/WsRSocketJavaBenchmark.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2015-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.rsocket.kotlin.benchmarks.java
+
+import io.rsocket.transport.*
+import io.rsocket.transport.netty.client.*
+import io.rsocket.transport.netty.server.*
+
+class WsRSocketJavaBenchmark : RSocketJavaBenchmark() {
+    override val serverTransport: ServerTransport<*> = WebsocketServerTransport.create(9000)
+    override val clientTransport: ClientTransport = WebsocketClientTransport.create(9000)
+}

benchmarks/rsocket-kotlin-0_16/build.gradle.kts

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2015-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import kotlinx.benchmark.gradle.*
+import org.gradle.kotlin.dsl.benchmark
+import rsocketbuild.*
+
+plugins {
+    id("rsocketbuild.multiplatform-benchmarks")
+}
+
+kotlin {
+    jvmTarget()
+
+    // no mingw because of cio
+    macosX64()
+    macosArm64()
+    linuxX64()
+
+    sourceSets {
+        commonMain.dependencies {
+            implementation(projects.benchmarksShared)
+            implementation("io.rsocket.kotlin:rsocket-transport-local:0.16.0")
+            implementation("io.rsocket.kotlin:rsocket-transport-ktor-tcp:0.16.0")
+        }
+    }
+}
+
+benchmark {
+    targets {
+        register("jvm") {
+            this as JvmBenchmarkTarget
+            jmhVersion = libs.versions.jmh.get()
+        }
+        register("macosArm64")
+        register("macosX64")
+        register("linuxX64")
+    }
+    registerBenchmarks("RSocketKotlin_0_16_", listOf("local", "ktorTcp")) { transport ->
+        if (transport == "local") {
+            param("dispatcher", "DEFAULT", "UNCONFINED")
+        }
+    }
+}

benchmarks/rsocket-kotlin-old/src/commonMain/kotlin/KtorTcpRSocketKotlinBenchmark.kt renamed to benchmarks/rsocket-kotlin-0_16/src/commonMain/kotlin/KtorTcpRSocketKotlin_0_16_Benchmark.kt

@@ -17,17 +17,18 @@
 package io.rsocket.kotlin.benchmarks.kotlin
 
 import io.ktor.network.sockets.*
+import io.rsocket.kotlin.benchmarks.*
 import io.rsocket.kotlin.transport.*
 import io.rsocket.kotlin.transport.ktor.tcp.*
 import kotlinx.benchmark.*
 import kotlinx.coroutines.*
 import kotlin.random.*
 
 @BenchmarkMode(Mode.Throughput)
-@Warmup(iterations = 3, time = 1)
-@Measurement(iterations = 3, time = 1)
+@Warmup(iterations = WARMUP, time = WARMUP_DURATION)
+@Measurement(iterations = ITERATION, time = ITERATION_DURATION)
 @State(Scope.Benchmark)
-class KtorTcpRSocketKotlinOldBenchmark : RSocketKotlinOldBenchmark() {
+class KtorTcpRSocketKotlin_0_16_Benchmark : RSocketKotlin_0_16_Benchmark() {
     @Param("0")
     override var payloadSize: Int = 0
 

benchmarks/rsocket-kotlin-old/src/commonMain/kotlin/LocalRSocketKotlinOldBenchmark.kt renamed to benchmarks/rsocket-kotlin-0_16/src/commonMain/kotlin/LocalRSocketKotlin_0_16_Benchmark.kt

@@ -16,19 +16,17 @@
 
 package io.rsocket.kotlin.benchmarks.kotlin
 
+import io.rsocket.kotlin.benchmarks.*
 import io.rsocket.kotlin.transport.*
 import io.rsocket.kotlin.transport.local.*
 import kotlinx.benchmark.*
 import kotlinx.coroutines.*
 
 @BenchmarkMode(Mode.Throughput)
-@Warmup(iterations = 3, time = 1)
-@Measurement(iterations = 3, time = 1)
+@Warmup(iterations = WARMUP, time = WARMUP_DURATION)
+@Measurement(iterations = ITERATION, time = ITERATION_DURATION)
 @State(Scope.Benchmark)
-class LocalRSocketKotlinOldBenchmark : RSocketKotlinOldBenchmark() {
-    @Param("0")
-    override var payloadSize: Int = 0
-
+class LocalRSocketKotlin_0_16_Benchmark : RSocketKotlin_0_16_Benchmark() {
     @Param("")
     var dispatcher: String = ""