Add benchmarks for Metadata collection (#1691)

Author: gjcairo

.github/workflows/ci.yaml

@@ -108,7 +108,7 @@ jobs:
       timeout-minutes: 20
     - name: Run Benchmarks
       working-directory: ./Performance/Benchmarks
-      run: swift package benchmark baseline check --check-absolute-path Thresholds/${{ matrix.swift-version }}/
+      run: swift package benchmark baseline check --no-progress --check-absolute-path Thresholds/${{ matrix.swift-version }}/
       timeout-minutes: 20
   integration-tests:
     strategy:

Package.swift

@@ -467,6 +467,11 @@ extension Product {
     name: grpcProductName,
     targets: [grpcTargetName]
   )
+    
+  static let grpcCore: Product = .library(
+    name: "_GRPCCore",
+    targets: ["GRPCCore"]
+  )
 
   static let cgrpcZlib: Product = .library(
     name: cgrpcZlibProductName,
@@ -490,6 +495,7 @@ let package = Package(
   name: grpcPackageName,
   products: [
     .grpc,
+    .grpcCore,
     .cgrpcZlib,
     .protocGenGRPCSwift,
     .grpcSwiftPlugin,

Performance/Benchmarks/Benchmarks/GRPCSwiftBenchmark/Benchmarks.swift

@@ -14,24 +14,119 @@
  * limitations under the License.
  */
 import Benchmark
-import Foundation
+import GRPCCore
 
 let benchmarks = {
-    Benchmark.defaultConfiguration = .init(
-        metrics: [
-            .mallocCountTotal,
-            .syscalls,
-            .readSyscalls,
-            .writeSyscalls,
-            .memoryLeaked,
-            .retainCount,
-            .releaseCount,
-        ]
-    )
-    
-    // async code is currently still quite flaky in the number of retain/release it does so we don't measure them today
-    var configWithoutRetainRelease = Benchmark.defaultConfiguration
-    configWithoutRetainRelease.metrics.removeAll(where: { $0 == .retainCount || $0 == .releaseCount })
-    
-    // Add Benchmarks here
+  Benchmark.defaultConfiguration = .init(
+      metrics: [
+          .mallocCountTotal,
+          .syscalls,
+          .readSyscalls,
+          .writeSyscalls,
+          .memoryLeaked,
+          .retainCount,
+          .releaseCount,
+      ]
+  )
+  
+  // async code is currently still quite flaky in the number of retain/release it does so we don't measure them today
+  var configWithoutRetainRelease = Benchmark.defaultConfiguration
+  configWithoutRetainRelease.metrics.removeAll(where: { $0 == .retainCount || $0 == .releaseCount })
+  
+  Benchmark("Metadata_Add_string") { benchmark in
+    for _ in benchmark.scaledIterations {
+      var metadata = Metadata()
+      for i in 0..<1000 {
+          metadata.addString("\(i)", forKey: "\(i)")
+      }
+    }
+  }
+  
+  Benchmark("Metadata_Add_binary") { benchmark in
+    let value: [UInt8] = [1, 2, 3]
+    for _ in benchmark.scaledIterations {
+      var metadata = Metadata()
+      
+      benchmark.startMeasurement()
+      for i in 0..<1000 {
+          metadata.addBinary(value, forKey: "\(i)")
+      }
+      benchmark.stopMeasurement()
+    }
+  }
+  
+  Benchmark("Metadata_Remove_values_for_key") { benchmark in
+    for _ in benchmark.scaledIterations {
+      var metadata = Metadata()
+      for i in 0..<1000 {
+          metadata.addString("value", forKey: "\(i)")
+      }
+      
+      benchmark.startMeasurement()
+      for i in 0..<1000 {
+          metadata.removeAllValues(forKey: "\(i)")
+      }
+      benchmark.stopMeasurement()
+    }
+  }
+  
+  Benchmark("Metadata_Iterate_all_values") { benchmark in
+    for _ in benchmark.scaledIterations {
+      var metadata = Metadata()
+      for i in 0..<1000 {
+        metadata.addString("value", forKey: "key")
+      }
+      
+      benchmark.startMeasurement()
+      for value in metadata["key"] {
+        blackHole(value)
+      }
+      benchmark.stopMeasurement()
+    }
+  }
+  
+  Benchmark("Metadata_Iterate_string_values") { benchmark in
+    for _ in benchmark.scaledIterations {
+      var metadata = Metadata()
+      for i in 0..<1000 {
+          metadata.addString("\(i)", forKey: "key")
+      }
+      
+      benchmark.startMeasurement()
+      for value in metadata[stringValues: "key"] {
+        blackHole(value)
+      }
+      benchmark.stopMeasurement()
+    }
+  }
+  
+  Benchmark("Metadata_Iterate_binary_values_when_only_binary_values_stored") { benchmark in
+    for _ in benchmark.scaledIterations {
+      var metadata = Metadata()
+      for i in 0..<1000 {
+        metadata.addBinary([1], forKey: "key")
+      }
+      
+      benchmark.startMeasurement()
+      for value in metadata[binaryValues: "key"] {
+        blackHole(value)
+      }
+      benchmark.stopMeasurement()
+    }
+  }
+  
+  Benchmark("Metadata_Iterate_binary_values_when_only_strings_stored") { benchmark in
+    for _ in benchmark.scaledIterations {
+      var metadata = Metadata()
+      for i in 0..<1000 {
+          metadata.addString("\(i)", forKey: "key")
+      }
+      
+      benchmark.startMeasurement()
+      for value in metadata[binaryValues: "key"] {
+        blackHole(value)
+      }
+      benchmark.stopMeasurement()
+    }
+  }
 }

Performance/Benchmarks/Package.swift

@@ -30,7 +30,7 @@ let package = Package(
             name: "GRPCSwiftBenchmark",
             dependencies: [
                 .product(name: "Benchmark", package: "package-benchmark"),
-                .product(name: "GRPC", package: "grpc-swift")
+                .product(name: "_GRPCCore", package: "grpc-swift")
             ],
             path: "Benchmarks/GRPCSwiftBenchmark",
             plugins: [

Performance/Benchmarks/Thresholds/5.7/GRPCSwiftBenchmark.Metadata_Add_binary.p90.json

@@ -0,0 +1,7 @@
+{
+  "mallocCountTotal" : 11,
+  "memoryLeaked" : 0,
+  "releaseCount" : 1011,
+  "retainCount" : 2000,
+  "syscalls" : 0
+}

Performance/Benchmarks/Thresholds/5.7/GRPCSwiftBenchmark.Metadata_Add_string.p90.json

@@ -0,0 +1,7 @@
+{
+  "mallocCountTotal" : 11,
+  "memoryLeaked" : 0,
+  "releaseCount" : 4012,
+  "retainCount" : 2000,
+  "syscalls" : 0
+}

Performance/Benchmarks/Thresholds/5.7/GRPCSwiftBenchmark.Metadata_Iterate_all_values.p90.json

@@ -0,0 +1,7 @@
+{
+  "mallocCountTotal" : 0,
+  "memoryLeaked" : 0,
+  "releaseCount" : 1005,
+  "retainCount" : 1005,
+  "syscalls" : 0
+}

Performance/Benchmarks/Thresholds/5.7/GRPCSwiftBenchmark.Metadata_Iterate_binary_values_when_only_binary_values_stored.p90.json

@@ -0,0 +1,7 @@
+{
+  "mallocCountTotal" : 0,
+  "memoryLeaked" : 0,
+  "releaseCount" : 1005,
+  "retainCount" : 1005,
+  "syscalls" : 0
+}

Performance/Benchmarks/Thresholds/5.7/GRPCSwiftBenchmark.Metadata_Iterate_binary_values_when_only_strings_stored.p90.json

@@ -0,0 +1,7 @@
+{
+  "mallocCountTotal" : 2000,
+  "memoryLeaked" : 0,
+  "releaseCount" : 4005,
+  "retainCount" : 2005,
+  "syscalls" : 0
+}

Performance/Benchmarks/Thresholds/5.7/GRPCSwiftBenchmark.Metadata_Iterate_string_values.p90.json

@@ -0,0 +1,7 @@
+{
+  "mallocCountTotal" : 0,
+  "memoryLeaked" : 0,
+  "releaseCount" : 1005,
+  "retainCount" : 1005,
+  "syscalls" : 0
+}