Push 'Add Benchmarks for metrics #106' over the finish line

.github/workflows/pull_request.yml

@@ -19,6 +19,12 @@ jobs:
       linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable"
       linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable"
 
+  benchmarks:
+    name: Benchmarks
+    uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
+    with:
+      benchmark_package_path: "Benchmarks"
+
   cxx-interop:
     name: Cxx interop
     uses: apple/swift-nio/.github/workflows/cxx_interop.yml@main

Benchmarks/.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+.build
+/.build
+/Packages
+xcuserdata/
+DerivedData/
+.swiftpm/configuration/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc
+.benchmarkBaselines/

Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift

@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Foundation
+import Prometheus
+
+let registry = PrometheusCollectorRegistry()
+
+public func makeLabels(_ idx: Int) -> [(String, String)] {
+    [
+        ("job", "api_server_\(idx)"),
+        ("handler", "/api/handler_\(idx)"),
+        ("status_code", "200"),
+        ("version", "\(idx).0.0"),
+    ]
+}
+
+let benchmarks = {
+    let ciMetrics: [BenchmarkMetric] = [
+        .mallocCountTotal
+    ]
+    let localMetrics = BenchmarkMetric.default
+
+    Benchmark.defaultConfiguration = .init(
+        metrics: ProcessInfo.processInfo.environment["CI"] != nil ? ciMetrics : localMetrics,
+        warmupIterations: 10,
+        scalingFactor: .kilo,
+        maxDuration: .seconds(5)
+    )
+
+    Benchmark("Counter - setup and increment") { benchmark in
+        runCounterBench(benchmark.scaledIterations)
+    }
+
+    Benchmark("Counter - increment only") { benchmark, run in
+        for _ in benchmark.scaledIterations {
+            run()
+        }
+    } setup: {
+        setupCounterBench()
+    }
+
+    Benchmark("Gauge") { benchmark in
+        runGaugeBench(benchmark.scaledIterations)
+    }
+
+    Benchmark("DurationHistogram") { benchmark in
+        runDurationHistogramBench(benchmark.scaledIterations)
+    }
+
+    Benchmark(
+        "RegistryEmit - 5000 metrics",
+        configuration: .init(scalingFactor: .one)
+    ) { benchmark, run in
+        for _ in benchmark.scaledIterations {
+            run()
+        }
+    } setup: {
+        setupRegistryExport(numberOfMetrics: 5000)
+    }
+}

Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Prometheus
+
+public func runCounterBench(_ iterations: Range<Int>) {
+    let ctr = registry.makeCounter(name: "counter_1", labels: makeLabels(1))
+    for _ in iterations {
+        blackHole(ctr.increment())
+    }
+}
+
+public func setupCounterBench() -> () -> Void {
+    let ctr = registry.makeCounter(name: "counter_2", labels: makeLabels(2))
+    return {
+        blackHole(ctr.increment())
+    }
+}

Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift

@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Prometheus
+
+public func runDurationHistogramBench(_ iterations: Range<Int>) {
+    let histogram = registry.makeDurationHistogram(
+        name: "histogram_1",
+        labels: makeLabels(3),
+        buckets: [
+            .milliseconds(100),
+            .milliseconds(250),
+            .milliseconds(500),
+            .seconds(1),
+        ]
+    )
+    for _ in iterations {
+        blackHole(histogram.record(Duration.milliseconds(400)))
+    }
+}

Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Prometheus
+
+public func runGaugeBench(_ iterations: Range<Int>) {
+    let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2))
+    for _ in iterations {
+        blackHole(gauge.increment())
+    }
+}
+
+public func setupGaugeBench() -> () -> Void {
+    let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2))
+    return {
+        blackHole(gauge.increment())
+    }
+}

Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift

@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Prometheus
+
+public func setupRegistryExport(numberOfMetrics: Int) -> () -> Void {
+    let registryExport = PrometheusCollectorRegistry()
+
+    var counterArray = [Counter]()
+    var gaugeArray = [Gauge]()
+    var buffer = [UInt8]()
+
+    let counterExportSize = 620_000
+    counterArray.reserveCapacity(numberOfMetrics)
+    gaugeArray.reserveCapacity(numberOfMetrics)
+    buffer.reserveCapacity(counterExportSize)
+
+    for i in 0..<(numberOfMetrics / 2) {
+        let counter = registryExport.makeCounter(name: "http_requests_total", labels: makeLabels(i))
+        counter.increment()
+        counterArray.append(counter)
+
+        let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: makeLabels(i))
+        gauge.increment()
+        gaugeArray.append(gauge)
+    }
+    return {
+        blackHole(registryExport.emit(into: &buffer))
+    }
+}

Benchmarks/Package.swift

@@ -0,0 +1,40 @@
+// swift-tools-version:5.9
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import PackageDescription
+
+let package = Package(
+    name: "benchmarks",
+    platforms: [
+        .macOS(.v13)
+    ],
+    dependencies: [
+        .package(path: "../"),
+        .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.29.0"),
+    ],
+    targets: [
+        .executableTarget(
+            name: "PrometheusBenchmarks",
+            dependencies: [
+                .product(name: "Benchmark", package: "package-benchmark"),
+                .product(name: "Prometheus", package: "swift-prometheus"),
+            ],
+            path: "Benchmarks/PrometheusBenchmarks",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark")
+            ]
+        )
+    ]
+)

Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_-_increment_only.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/5.10/PrometheusBenchmarks.DurationHistogram.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 2
+}

Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Gauge.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/5.10/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/6.0/PrometheusBenchmarks.Counter_-_increment_only.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/6.0/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/6.0/PrometheusBenchmarks.DurationHistogram.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/6.0/PrometheusBenchmarks.Gauge.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/6.0/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/6.1/PrometheusBenchmarks.Counter_-_increment_only.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/6.1/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/6.1/PrometheusBenchmarks.DurationHistogram.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/6.1/PrometheusBenchmarks.Gauge.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/6.1/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/nightly-main/PrometheusBenchmarks.Counter_-_increment_only.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/nightly-main/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/nightly-main/PrometheusBenchmarks.DurationHistogram.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 2
+}

Benchmarks/Thresholds/nightly-main/PrometheusBenchmarks.Gauge.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/nightly-main/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/nightly-next/PrometheusBenchmarks.Counter_-_increment_only.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/nightly-next/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/nightly-next/PrometheusBenchmarks.DurationHistogram.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/nightly-next/PrometheusBenchmarks.Gauge.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}