Add input facilities to benchmarks and start making serious benchmarks

Author: Antoine Brunner

bench-run/inputs/concat.txt

@@ -0,0 +1 @@
+sizes:0 0,0 5,0 10,0 15,0 20,0 25,0 30,5 0,5 5,5 10,5 15,5 20,5 25,5 30,10 0,10 5,10 10,10 15,10 20,10 25,10 30,15 0,15 5,15 10,15 15,15 20,15 25,15 30,20 0,20 5,20 10,20 15,20 20,20 25,20 30,25 0,25 5,25 10,25 15,25 20,25 25,25 30,30 0,30 5,30 10,30 15,30 20,30 25,30 30

bench-run/outputs/concat.csv

@@ -1,18 +1,13 @@
 package dotty.tools.benchmarks
 
-import dotty.tools.dotc._
-import core.Contexts.Context
-import dotty.tools.FatalError
-import reporting._
-
 import org.openjdk.jmh.results.RunResult
 import org.openjdk.jmh.runner.Runner
 import org.openjdk.jmh.runner.options.OptionsBuilder
 import org.openjdk.jmh.annotations._
 import org.openjdk.jmh.results.format._
 import java.util.concurrent.TimeUnit
 
-import scala.runtime.DynamicTuple
+import scala.io.Source
 
 object Bench {
   def main(args: Array[String]): Unit = {
@@ -23,9 +18,9 @@ object Bench {
     val forks = if (intArgs.length > 2) intArgs(2).toInt else 1
 
     val benchmarks = if (args1.length > 0) args1(0) else ".*"
-    val outputFile = if (args1.length > 1) args1(1) else "output.csv"
+    val outputFile = if (args1.length > 2) args1(2) else "output.csv"
 
-    val opts = new OptionsBuilder()
+    var builder = new OptionsBuilder()
                .shouldFailOnError(true)
                .jvmArgs("-Xms2G", "-Xmx2G")
                .mode(Mode.AverageTime)
@@ -35,10 +30,21 @@ object Bench {
                .forks(forks)
                .include(benchmarks)
                .resultFormat(ResultFormatType.CSV)
-               .result("results/" ++ outputFile)
-               .build
+               .result(outputFile)
+
+    if (args1.length > 1) {
+      for ((param, values) <- paramsFromFile(args1(1)))
+        builder = builder.param(param, values: _*)
+    }
+
+    val runner = new Runner(builder.build) // full access to all JMH features, you can also provide a custom output Format here
+    runner.run // actually run the benchmarks
+  }
 
-    val runner = new Runner(opts) // full access to all JMH features, you can also provide a custom output Format here
-    runner.run() // actually run the benchmarks
+  def paramsFromFile(file: String): Array[(String, Array[String])] = {
+    Source.fromFile(file).getLines.toArray.map { l =>
+      val Array(param, values) = l split ':'
+      (param, values split ',')
+    }
   }
 }

bench-run/src/main/scala/Benchmarks.scala renamed to bench-run/src/main/scala/Main.scala

@@ -0,0 +1,35 @@
+package dotty.tools.benchmarks.tuples
+
+import org.openjdk.jmh.annotations._
+import scala.runtime.DynamicTuple
+
+@State(Scope.Thread)
+class ApplyBenchmarks {
+  @Param(Array("1 0"))
+  var sizeAndIndex: String = _
+  var tuple: NonEmptyTuple = _
+  var index: Int = _
+
+  @Setup
+  def setup(): Unit = {
+    val size = sizeAndIndex.split(' ')(0).toInt
+    index = sizeAndIndex.split(' ')(1).toInt
+    tuple = "elem" *: ()
+
+    for (i <- 1 until size)
+      tuple = "string" *: tuple
+  }
+
+  @Benchmark
+  def baseline(): Unit = {}
+
+  @Benchmark
+  def normal(): Any = {
+    tuple(index)
+  }
+
+  @Benchmark
+  def inlined(): Any = {
+    DynamicTuple.dynamicApply(tuple, index)
+  }
+}

bench-run/src/main/scala/tuples/Apply.scala

@@ -0,0 +1,64 @@
+package dotty.tools.benchmarks.tuples
+
+import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.infra.Blackhole
+import scala.runtime.DynamicTuple
+
+@State(Scope.Thread)
+class Concat {
+  @Param(Array("0 0"))
+  var sizes: String = _
+  var tuple1: Tuple = _
+  var tuple2: Tuple = _
+  var array1: Array[Object] = _
+  var array2: Array[Object] = _
+
+  def tupleOfSize(n: Int): Tuple = {
+    var t: Tuple = ()
+    for (i <- 1 to n)
+      t = "elem" *: t
+    t
+  }
+
+  @Setup
+  def setup(): Unit = {
+    val size1 = sizes.split(' ')(0).toInt
+    val size2 = sizes.split(' ')(1).toInt
+    tuple1 = tupleOfSize(size1)
+    tuple1 = tupleOfSize(size2)
+    array1 = Array.fill(size1)("elem")
+    array2 = Array.fill(size2)("elem")
+  }
+
+  @Benchmark
+  def baseline(): Unit = {}
+
+  @Benchmark
+  def normal(): Tuple = {
+   tuple1 ++ tuple2
+  }
+
+  @Benchmark
+  def inlined(): Tuple = {
+    DynamicTuple.dynamicConcat(tuple1, tuple2)
+  }
+
+  // This part is here to try and measure the overhead of tranforming tuples to arrays, then concatenating
+  // the array, and then transforming back to a tuple
+  @Benchmark
+  def toArray(bh: Blackhole): Unit = {
+    bh.consume(DynamicTuple.dynamicToArray(tuple1))
+    bh.consume(DynamicTuple.dynamicToArray(tuple2))
+  }
+
+  @Benchmark
+  def arrayConcat(): Array[Object] = {
+    array1 ++ array2
+  }
+
+  @Benchmark
+  def fromArray(bh: Blackhole): Unit = {
+    bh.consume(DynamicTuple.dynamicFromArray[Tuple](array1))
+    bh.consume(DynamicTuple.dynamicFromArray[Tuple](array2))
+  }
+}

bench-run/src/main/scala/tuples/Concat.scala

@@ -30,7 +30,6 @@ object Bench {
     val iterations = if (intArgs.length > 1) intArgs(1).toInt else 20
     val forks = if (intArgs.length > 2) intArgs(2).toInt else 1
 
-
     import File.{ separator => sep }
 
     val args2 = args1.map { arg =>