diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index b09ca0d56..2bcc1accc 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -20,3 +20,35 @@ jobs: with: license_header_check_project_name: "Swift.org" api_breakage_check_allowlist_path: "api-breakages.txt" + performance_test: + runs-on: ubuntu-latest + container: + image: swift:latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Download performance comparison script + run: | + apt-get update && apt-get install -y curl + curl -s https://raw.githubusercontent.com/ahoppen/swift-format/refs/heads/main/.github/workflows/scripts/compare-performance-measurements.swift > /tmp/compare-performance-measurements.swift + - name: Measure performance with PR changes + id: pr_performance + run: | + OUTPUT=$(echo "Random: $(random)") + echo "output=$OUTPUT" >> "$GITHUB_OUTPUT" + - name: Measure baseline performance + id: baseline_performance + run: | + OUTPUT=$(echo "Random: $(random)") + echo "output=$OUTPUT" >> "$GITHUB_OUTPUT" + - name: Post comment if performance changed + run: | + if ! OUTPUT=$(swift /tmp/compare-performance-measurements.swift "${{ steps.baseline_performance.outputs.output }}" "${{ steps.pr_performance.outputs.output }}" 0.5); then + gh pr comment --body "$OUTPUT" + else + echo "No significant performance change detected" + echo "$OUTPUT" + fi + diff --git a/.github/workflows/scripts/compare-performance-measurements.swift b/.github/workflows/scripts/compare-performance-measurements.swift new file mode 100644 index 000000000..132388a19 --- /dev/null +++ b/.github/workflows/scripts/compare-performance-measurements.swift @@ -0,0 +1,90 @@ +import Foundation + +func printToStderr(_ value: String) { + fputs(value, stderr) +} + +func extractMeasurements(output: String) -> [String: Double] { + var measurements: [String: Double] = [:] + for line in output.split(separator: "\n") { + guard let colonPosition = line.lastIndex(of: ":") else { + printToStderr("Ignoring following measurement line because it doesn't contain a colon: \(line)") + continue + } + let beforeColon = String(line[.. Double { + return (self * pow(10, Double(decimalDigits))).rounded() / pow(10, Double(decimalDigits)) + } +} + +func run( + baselinePerformanceOutput: String, + changedPerformanceOutput: String, + sensitivityPercentage: Double +) -> (output: String, hasDetectedSignificantChange: Bool) { + let baselineMeasurements = extractMeasurements(output: baselinePerformanceOutput) + let changedMeasurements = extractMeasurements(output: changedPerformanceOutput) + + var hasDetectedSignificantChange = false + var output = "" + for (measurementName, baselineValue) in baselineMeasurements.sorted(by: { $0.key < $1.key }) { + guard let changedValue = changedMeasurements[measurementName] else { + output += "🛑 \(measurementName) not present after changes\n" + continue + } + let differencePercentage = (changedValue - baselineValue) / baselineValue * 100 + let rawMeasurementsText = "(baseline: \(baselineValue), after changes: \(changedValue))" + if differencePercentage < -sensitivityPercentage { + output += + "🎉 \(measurementName) improved by \(-differencePercentage.round(toDecimalDigits: 3))% \(rawMeasurementsText)\n" + hasDetectedSignificantChange = true + } else if differencePercentage > sensitivityPercentage { + output += + "⚠️ \(measurementName) regressed by \(differencePercentage.round(toDecimalDigits: 3))% \(rawMeasurementsText)\n" + hasDetectedSignificantChange = true + } else { + output += + "➡️ \(measurementName) did not change significantly with \(differencePercentage.round(toDecimalDigits: 3))% \(rawMeasurementsText)\n" + } + } + return (output, hasDetectedSignificantChange) +} + +guard CommandLine.arguments.count > 2 else { + print("Expected at least two parameters: The baseline performance output and the changed performance output") + exit(1) +} + +let baselinePerformanceOutput = CommandLine.arguments[1] +let changedPerformanceOutput = CommandLine.arguments[2] + +let sensitivityPercentage = + if CommandLine.arguments.count > 3, let percentage = Double(CommandLine.arguments[3]) { + percentage + } else { + 1.0 /* percent */ + } + +let (output, hasDetectedSignificantChange) = run( + baselinePerformanceOutput: baselinePerformanceOutput, + changedPerformanceOutput: changedPerformanceOutput, + sensitivityPercentage: sensitivityPercentage +) + +print(output) +if hasDetectedSignificantChange { + exit(1) +}