Add new comparison features

- Compare with NS regular expression
- Save comparison results
- Save and compare estimated compile times

Author: rctcwyvrn

Sources/RegexBenchmark/Benchmark.swift

@@ -89,6 +89,9 @@ struct InputListNSBenchmark: RegexBenchmark {
 
 /// A benchmark meant to be ran across multiple engines
 struct CrossBenchmark {
+  /// Suffix added onto NSRegularExpression benchmarks
+  static let nsSuffix = "_NS"
+  
   /// The base name of the benchmark
   var baseName: String
 
@@ -127,7 +130,7 @@ struct CrossBenchmark {
           target: input))
       runner.register(
         NSBenchmark(
-          name: baseName + "Whole_NS",
+          name: baseName + "Whole" + CrossBenchmark.nsSuffix,
           regex: nsRegex,
           type: .first,
           target: input))
@@ -140,7 +143,7 @@ struct CrossBenchmark {
           target: input))
       runner.register(
         NSBenchmark(
-          name: baseName + "All_NS",
+          name: baseName + "All" + CrossBenchmark.nsSuffix,
           regex: nsRegex,
           type: .allMatches,
           target: input))
@@ -153,7 +156,7 @@ struct CrossBenchmark {
             target: input))
         runner.register(
           NSBenchmark(
-            name: baseName + "First_NS",
+            name: baseName + "First" + CrossBenchmark.nsSuffix,
             regex: nsRegex,
             type: .first,
             target: input))

Sources/RegexBenchmark/BenchmarkChart.swift

@@ -15,14 +15,7 @@ import Charts
 import SwiftUI
 
 struct BenchmarkChart: View {
-  struct Comparison: Identifiable {
-    var id = UUID()
-    var name: String
-    var baseline: BenchmarkResult
-    var latest: BenchmarkResult
-  }
-
-  var comparisons: [Comparison]
+  var comparisons: [BenchmarkResult.Comparison]
 
   var body: some View {
     VStack(alignment: .leading) {
@@ -94,7 +87,7 @@ struct BenchmarkChart: View {
 }
 
 struct BenchmarkResultApp: App {
-  static var comparisons: [BenchmarkChart.Comparison]?
+  static var comparisons: [BenchmarkResult.Comparison]?
 
   var body: some Scene {
     WindowGroup {

Sources/RegexBenchmark/BenchmarkResults.swift

@@ -0,0 +1,224 @@
+import Foundation
+
+extension BenchmarkRunner {
+  func save(to savePath: String) throws {
+    let url = URL(fileURLWithPath: savePath, isDirectory: false)
+    let parent = url.deletingLastPathComponent()
+    if !FileManager.default.fileExists(atPath: parent.path) {
+      try! FileManager.default.createDirectory(atPath: parent.path, withIntermediateDirectories: true)
+    }
+    print("Saving result to \(url.path)")
+    try results.save(to: url)
+  }
+    
+  func compare(against compareFilePath: String, showChart: Bool, saveTo: String?) throws {
+    let compareFileURL = URL(fileURLWithPath: compareFilePath)
+    let compareResult = try SuiteResult.load(from: compareFileURL)
+    let compareFile = compareFileURL.lastPathComponent
+
+    let comparisons = results
+      .compare(with: compareResult)
+      .filter({!$0.name.contains("_NS")})
+      .filter({$0.diff != nil})
+    displayComparisons(comparisons, showChart, against: "saved benchmark result " + compareFile)
+    if let saveFile = saveTo {
+      try saveComparisons(comparisons, path: saveFile)
+    }
+    
+    let compileTimeComparisons = results
+      .compareCompileTimes(with: compareResult)
+      .filter({!$0.name.contains("_NS")})
+      .filter({$0.diff != nil})
+    print("Comparing estimated compile times")
+    displayComparisons(compileTimeComparisons, false, against: "saved benchmark result " + compareFile)
+  }
+  
+  func compareWithNS(showChart: Bool, saveTo: String?) throws {
+    let comparisons = results.compareWithNS().filter({$0.diff != nil})
+    displayComparisons(comparisons, showChart, against: "NSRegularExpression (via CrossBenchmark)")
+    if let saveFile = saveTo {
+      try saveComparisons(comparisons, path: saveFile)
+    }
+  }
+  
+  func displayComparisons(_ comparisons: [BenchmarkResult.Comparison], _ showChart: Bool, against: String) {
+    let regressions = comparisons.filter({$0.diff!.seconds > 0})
+      .sorted(by: {(a,b) in a.diff!.seconds > b.diff!.seconds})
+    let improvements = comparisons.filter({$0.diff!.seconds < 0})
+      .sorted(by: {(a,b) in a.diff!.seconds < b.diff!.seconds})
+    
+    print("Comparing against \(against)")
+    print("=== Regressions ======================================================================")
+    for item in regressions {
+      print(item)
+    }
+    
+    print("=== Improvements =====================================================================")
+    for item in improvements {
+      print(item)
+    }
+
+    #if os(macOS)
+    if showChart {
+      print("""
+        === Comparison chart =================================================================
+        Press Control-C to close...
+        """)
+      BenchmarkResultApp.comparisons = {
+        return comparisons.sorted {
+          let delta0 = Float($0.latest.median.seconds - $0.baseline.median.seconds)
+            / Float($0.baseline.median.seconds)
+          let delta1 = Float($1.latest.median.seconds - $1.baseline.median.seconds)
+            / Float($1.baseline.median.seconds)
+          return delta0 > delta1
+        }
+      }()
+      BenchmarkResultApp.main()
+    }
+    #endif
+  }
+  
+  func saveComparisons(_ comparisons: [BenchmarkResult.Comparison], path: String) throws {
+    let url = URL(fileURLWithPath: path, isDirectory: false)
+    let parent = url.deletingLastPathComponent()
+    if !FileManager.default.fileExists(atPath: parent.path) {
+      try! FileManager.default.createDirectory(atPath: parent.path, withIntermediateDirectories: true)
+    }
+    
+    var contents = "name,baseline,latest,diff,percentage\n"
+    for comparison in comparisons {
+      contents += comparison.asCsv + "\n"
+    }
+    print("Saving comparisons as .csv to \(path)")
+    try contents.write(to: url, atomically: true, encoding: String.Encoding.utf8)
+  }
+}
+
+struct BenchmarkResult: Codable {
+  let median: Time
+  let estimatedCompileTime: Time
+  let stdev: Double
+  let samples: Int
+
+  init(_ initialRunTime: Time, _ median: Time, _ stdev: Double, _ samples: Int) {
+    self.estimatedCompileTime = initialRunTime - median
+    self.median = median
+    self.stdev = stdev
+    self.samples = samples
+  }
+}
+
+extension BenchmarkResult {
+  struct Comparison: Identifiable, CustomStringConvertible {
+    var id = UUID()
+    var name: String
+    var baseline: BenchmarkResult
+    var latest: BenchmarkResult
+    var diffCompileTimes: Bool = false
+    
+    var diff: Time? {
+      if diffCompileTimes {
+        return latest.estimatedCompileTime - baseline.estimatedCompileTime
+      }
+      if Stats.tTest(baseline, latest) {
+        return latest.median - baseline.median
+      }
+      return nil
+    }
+    
+    var description: String {
+      guard let diff = diff else {
+        return "- \(name) N/A"
+      }
+      let oldVal: Time
+      let newVal: Time
+      if diffCompileTimes {
+        oldVal = baseline.estimatedCompileTime
+        newVal = latest.estimatedCompileTime
+      } else {
+        oldVal = baseline.median
+        newVal = latest.median
+      }
+      let percentage = (1000 * diff.seconds / oldVal.seconds).rounded()/10
+      let len = max(40 - name.count, 1)
+      let nameSpacing = String(repeating: " ", count: len)
+      return "- \(name)\(nameSpacing)\(newVal)\t\(oldVal)\t\(diff)\t\t\(percentage)%"
+    }
+    
+    var asCsv: String {
+      guard let diff = diff else {
+        return "\(name),N/A"
+      }
+      let oldVal: Time
+      let newVal: Time
+      if diffCompileTimes {
+        oldVal = baseline.estimatedCompileTime
+        newVal = latest.estimatedCompileTime
+      } else {
+        oldVal = baseline.median
+        newVal = latest.median
+      }
+      let percentage = (1000 * diff.seconds / oldVal.seconds).rounded()/10
+      return "\"\(name)\",\(newVal.seconds),\(oldVal.seconds),\(diff.seconds),\(percentage)%"
+    }
+  }
+}
+
+struct SuiteResult {
+  var results: [String: BenchmarkResult] = [:]
+  
+  mutating func add(name: String, result: BenchmarkResult) {
+    results.updateValue(result, forKey: name)
+  }
+  
+  /// Compares with the given SuiteResult
+  func compare(with other: SuiteResult) -> [BenchmarkResult.Comparison] {
+    var comparisons: [BenchmarkResult.Comparison] = []
+    for item in results {
+      if let otherVal = other.results[item.key] {
+        comparisons.append(
+          .init(name: item.key, baseline: item.value, latest: otherVal))
+      }
+    }
+    return comparisons
+  }
+  
+  /// Compares with the NSRegularExpression benchmarks generated by CrossBenchmark
+  func compareWithNS() -> [BenchmarkResult.Comparison] {
+    var comparisons: [BenchmarkResult.Comparison] = []
+    for item in results {
+      let key = item.key + CrossBenchmark.nsSuffix
+      if let nsResult = results[key] {
+        comparisons.append(
+          .init(name: item.key, baseline: nsResult, latest: item.value))
+      }
+    }
+    return comparisons
+  }
+  
+  /// Compares the estimated compile times
+  func compareCompileTimes(with other: SuiteResult) -> [BenchmarkResult.Comparison] {
+    var comparisons: [BenchmarkResult.Comparison] = []
+    for item in results {
+      if let otherVal = other.results[item.key] {
+        comparisons.append(
+          .init(name: item.key, baseline: item.value, latest: otherVal, diffCompileTimes: true))
+      }
+    }
+    return comparisons
+  }
+}
+
+extension SuiteResult: Codable {
+  func save(to url: URL) throws {
+    let encoder = JSONEncoder()
+    let data = try encoder.encode(self)
+    try data.write(to: url, options: .atomic)
+  }
+  
+  static func load(from url: URL) throws -> SuiteResult {
+    let decoder = JSONDecoder()
+    let data = try Data(contentsOf: url)
+    return try decoder.decode(SuiteResult.self, from: data)
+  }
+}