Add support for ordo-one/package-benchmark (#1678)

Author: gjcairo

.github/workflows/ci.yaml

@@ -50,6 +50,7 @@ jobs:
       matrix:
         include:
           - image: swiftlang/swift:nightly-jammy
+            swift-version: main
             env:
               MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_10_requests: 323000
               MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_1_request: 161000
@@ -60,6 +61,7 @@ jobs:
               MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_client: 170000
               MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_server: 170000
           - image: swift:5.9-jammy
+            swift-version: 5.9
             env:
               MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_10_requests: 323000
               MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_1_request: 161000
@@ -70,6 +72,7 @@ jobs:
               MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_client: 170000
               MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_server: 170000
           - image: swift:5.8-focal
+            swift-version: 5.8
             env:
               MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_10_requests: 323000
               MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_1_request: 161000
@@ -80,6 +83,7 @@ jobs:
               MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_client: 170000
               MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_server: 170000
           - image: swift:5.7-focal
+            swift-version: 5.7
             env:
               MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_10_requests: 323000
               MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_1_request: 161000
@@ -99,6 +103,13 @@ jobs:
       run: ./Performance/allocations/test-allocation-counts.sh
       env: ${{ matrix.env }}
       timeout-minutes: 20
+    - name: Install jemalloc for benchmarking
+      run: apt update && apt-get install -y libjemalloc-dev
+      timeout-minutes: 20
+    - name: Run Benchmarks
+      working-directory: ./Performance/Benchmarks
+      run: swift package benchmark baseline check --check-absolute-path Thresholds/${{ matrix.swift-version }}/
+      timeout-minutes: 20
   integration-tests:
     strategy:
       fail-fast: false

.gitignore

@@ -22,3 +22,4 @@ dev/codegen-tests/**/generated/*
 /scripts/.swift-format-source/
 Package.resolved
 *.out.*
+/Performance/Benchmarks/.benchmarkBaselines/

Performance/Benchmarks/Benchmarks/GRPCSwiftBenchmark/Benchmarks.swift

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023, gRPC Authors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Benchmark
+import Foundation
+
+let benchmarks = {
+    Benchmark.defaultConfiguration = .init(
+        metrics: [
+            .mallocCountTotal,
+            .syscalls,
+            .readSyscalls,
+            .writeSyscalls,
+            .memoryLeaked,
+            .retainCount,
+            .releaseCount,
+        ]
+    )
+    
+    // async code is currently still quite flaky in the number of retain/release it does so we don't measure them today
+    var configWithoutRetainRelease = Benchmark.defaultConfiguration
+    configWithoutRetainRelease.metrics.removeAll(where: { $0 == .retainCount || $0 == .releaseCount })
+    
+    // Add Benchmarks here
+}

Performance/Benchmarks/Package.swift

@@ -0,0 +1,41 @@
+// swift-tools-version: 5.7
+/*
+ * Copyright 2023, gRPC Authors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PackageDescription
+
+let package = Package(
+    name: "benchmarks",
+    platforms: [
+        .macOS(.v13),
+    ],
+    dependencies: [
+        .package(path: "../../"),
+        .package(url: "https://github.com/ordo-one/package-benchmark", from: "1.11.2")
+    ],
+    targets: [
+        .executableTarget(
+            name: "GRPCSwiftBenchmark",
+            dependencies: [
+                .product(name: "Benchmark", package: "package-benchmark"),
+                .product(name: "GRPC", package: "grpc-swift")
+            ],
+            path: "Benchmarks/GRPCSwiftBenchmark",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark")
+            ]
+        ),
+    ]
+)

README.md

@@ -140,6 +140,25 @@ The `docs` directory contains documentation, including:
 - How to configure keepalive in [`docs/keepalive.md`][docs-keepalive]
 - Support for Apple Platforms and NIO Transport Services in
   [`docs/apple-platforms.md`][docs-apple]
+  
+## Benchmarks
+
+Benchmarks for `grpc-swift` are in a separate Swift Package in the `Performance/Benchmarks` subfolder of this repository. 
+They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin.
+Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics.
+An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`. 
+Afterwards you can run the benchmarks from CLI by going to the `Performance/Benchmarks` subfolder (e.g. `cd Performance/Benchmarks`) and invoking:
+```
+swift package benchmark
+```
+
+Profiling benchmarks or building the benchmarks in release mode in Xcode with `jemalloc` is currently not supported and requires disabling `jemalloc`. 
+Make sure Xcode is closed and then open it from the CLI with the `BENCHMARK_DISABLE_JEMALLOC=true` environment variable set e.g.:
+```
+BENCHMARK_DISABLE_JEMALLOC=true xed .
+```
+
+For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark). 
 
 ## Security