Add option to show benchmark charts

Author: rxwei

Sources/RegexBenchmark/BenchmarkChart.swift

@@ -0,0 +1,112 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+//
+//===----------------------------------------------------------------------===//
+
+#if os(macOS)
+
+import Charts
+import SwiftUI
+
+struct BenchmarkChart: View {
+  struct Comparison: Identifiable {
+    var id = UUID()
+    var name: String
+    var baseline: BenchmarkResult
+    var latest: BenchmarkResult
+  }
+
+  var comparisons: [Comparison]
+
+  var body: some View {
+    VStack(alignment: .leading) {
+      ForEach(comparisons) { comparison in
+        let new = comparison.latest.median.seconds
+        let old = comparison.baseline.median.seconds
+        Chart {
+          chartBody(
+            name: comparison.name,
+            new: new,
+            old: old,
+            sampleCount: comparison.latest.samples)
+        }
+        .chartXAxis {
+          AxisMarks { value in
+            AxisTick()
+            AxisValueLabel {
+              Text(String(format: "%.5fs", value.as(Double.self)!))
+            }
+          }
+        }
+        .chartYAxis {
+          AxisMarks { value in
+            AxisGridLine()
+            AxisValueLabel {
+              HStack {
+                Text(value.as(String.self)!)
+                let delta = (new - old) / old * 100
+                Text(String(format: "%+.2f%%", delta))
+                  .foregroundColor(delta <= 0 ? .green : .yellow)
+              }
+            }
+          }
+        }
+        .frame(idealHeight: 60)
+      }
+    }
+  }
+
+  @ChartContentBuilder
+  func chartBody(
+    name: String,
+    new: TimeInterval,
+    old: TimeInterval,
+    sampleCount: Int
+  ) -> some ChartContent {
+    // Baseline bar
+    BarMark(
+      x: .value("Time", old),
+      y: .value("Name", "\(name) (\(sampleCount) samples)"))
+    .position(by: .value("Kind", "Baseline"))
+    .foregroundStyle(.gray)
+
+    // Latest result bar
+    BarMark(
+      x: .value("Time", new),
+      y: .value("Name", "\(name) (\(sampleCount) samples)"))
+    .position(by: .value("Kind", "Latest"))
+    .foregroundStyle(LinearGradient(
+      colors: [.accentColor, new - old <= 0 ? .green : .yellow],
+      startPoint: .leading,
+      endPoint: .trailing))
+
+    // Comparison
+    RuleMark(x: .value("Time", new))
+    .foregroundStyle(.gray)
+    .lineStyle(.init(lineWidth: 0.5, dash: [2]))
+  }
+}
+
+struct BenchmarkResultApp: App {
+  static var comparisons: [BenchmarkChart.Comparison]?
+
+  var body: some Scene {
+    WindowGroup {
+      if let comparisons = Self.comparisons {
+        ScrollView {
+          BenchmarkChart(comparisons: comparisons)
+        }
+      } else {
+        Text("No data")
+      }
+    }
+  }
+}
+
+#endif

Sources/RegexBenchmark/BenchmarkRunner.swift

@@ -89,7 +89,7 @@ extension BenchmarkRunner {
     try results.save(to: url)
   }
 
-  func compare(against compareFilePath: String) throws {
+  func compare(against compareFilePath: String, showChart: Bool) throws {
     let compareFileURL = URL(fileURLWithPath: compareFilePath)
     let compareResult = try SuiteResult.load(from: compareFileURL)
     let compareFile = compareFileURL.lastPathComponent
@@ -121,14 +121,40 @@ extension BenchmarkRunner {
     for item in improvements {
       printComparison(name: item.key, diff: item.value)
     }
+
+    #if os(macOS)
+    if showChart {
+      print("""
+        === Comparison chart =================================================================
+        Press Control-C to close...
+        """)
+      BenchmarkResultApp.comparisons = {
+        var comparisons: [BenchmarkChart.Comparison] = []
+        for (name, baseline) in compareResult.results {
+          if let latest = results.results[name] {
+            comparisons.append(
+              .init(name: name, baseline: baseline, latest: latest))
+          }
+        }
+        return comparisons.sorted {
+          let delta0 = Float($0.latest.median.seconds - $0.baseline.median.seconds)
+            / Float($0.baseline.median.seconds)
+          let delta1 = Float($1.latest.median.seconds - $1.baseline.median.seconds)
+            / Float($1.baseline.median.seconds)
+          return delta0 > delta1
+        }
+      }()
+      BenchmarkResultApp.main()
+    }
+    #endif
   }
 }
 
 struct BenchmarkResult: Codable {
   let median: Time
   let stdev: Double
   let samples: Int
-  
+
   init(_ median: Time, _ stdev: Double, _ samples: Int) {
     self.median = median
     self.stdev = stdev

Sources/RegexBenchmark/CLI.swift

@@ -17,6 +17,9 @@ struct Runner: ParsableCommand {
   @Option(help: "The result file to compare against")
   var compare: String?
 
+  @Flag(help: "Show comparison chart")
+  var showChart: Bool = false
+
   @Flag(help: "Quiet mode")
   var quiet = false
 
@@ -40,12 +43,12 @@ struct Runner: ParsableCommand {
         runner.suite = runner.suite.filter { b in !b.name.contains("NS") }
       }
       runner.run()
-      if let compareFile = compare {
-        try runner.compare(against: compareFile)
-      }
       if let saveFile = save {
         try runner.save(to: saveFile)
       }
+      if let compareFile = compare {
+        try runner.compare(against: compareFile, showChart: showChart)
+      }
     }
   }
 }