From 76ebbbf3be5c9d31581566ef8722b453095e2e06 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 30 Jul 2025 02:02:42 -0400 Subject: [PATCH 01/12] Set up pipeline to generate perf comp comment during a PR run --- .evergreen/config.yml | 2 ++ .gitignore | 4 ++++ etc/perf-pr-comment.sh | 16 +++++++++++++++- internal/cmd/perfcomp/main.go | 20 +++++++++++++++----- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index cd1886cbc2..67664eb5b3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -292,6 +292,8 @@ functions: binary: bash env: VERSION_ID: ${version_id} + BASE_SHA: "${revision}" + HEAD_SHA: "${github_commit}" include_expansions_in_env: [PERF_URI_PRIVATE_ENDPOINT] args: [*task-runner, perf-pr-comment] diff --git a/.gitignore b/.gitignore index ef128dd7ff..11db098a16 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,10 @@ internal/cmd/compilecheck/compilecheck.so api-report.md api-report.txt +# Ignore perf report files +perf-report.md +perf-report.txt + # Ignore secrets files secrets-expansion.yml secrets-export.sh diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index ab7b9b8d23..b39d24a3ec 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -4,6 +4,20 @@ set -eux +# Generate perf report. pushd ./internal/cmd/perfcomp >/dev/null || exist -GOWORK=off go run main.go --project="mongo-go-driver" ${VERSION_ID} +GOWORK=off go run main.go --project="mongo-go-driver" ${VERSION_ID} > ./parseperfcomp/perf-report.txt popd >/dev/null + +if [[ -n "${BASE_SHA+set}" && -n "${HEAD_SHA+set}" && "$BASE_SHA" != "$HEAD_SHA" ]]; then + # Parse and generate perf comparison comment. + go run ./internal/cmd/perfcomp/parseperfcomp/main.go + # Make the PR comment. + # target=$DRIVERS_TOOLS/.evergreen/github_app/create_or_modify_comment.sh + # bash $target -m "## 👋GoDriver Performance" -c "$(pwd)/perf-report.md" -h $HEAD_SHA -o "mongodb" -n "mongo-go-driver" +else + # Skip comment if it isn't a PR run. + echo "Skipping Perf PR comment" +fi + +rm ./internal/cmd/perfcomp/parseperfcomp/perf-report.txt diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index e9c55d6d1f..0424c5ff92 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -166,7 +166,19 @@ func main() { if err != nil { log.Fatalf("Error getting energy statistics: %v", err) } - log.Println(generatePRComment(allEnergyStats, version)) + + // Log energy stats output + prComment := generatePRComment(allEnergyStats, version) + log.Println("👋GoDriver Performance") + log.Println(prComment) + + // Save for PR comment if it is a PR run + commitSHA := os.Getenv("HEAD_SHA") + if commitSHA != "" { + fmt.Printf("Version ID: %s\n", version) + fmt.Printf("Commit SHA: %s\n", commitSHA) // Use fmt to print to stdout + fmt.Println(prComment) + } } func findRawData(ctx context.Context, project string, version string, coll *mongo.Collection) ([]RawData, error) { @@ -305,8 +317,7 @@ func getEnergyStatsForAllBenchMarks(ctx context.Context, patchRawData []RawData, func generatePRComment(energyStats []*EnergyStats, version string) string { var comment strings.Builder - comment.WriteString("# 👋GoDriver Performance\n") - fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n", version) + fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n\n", version) w := tabwriter.NewWriter(&comment, 0, 0, 1, ' ', 0) fmt.Fprintln(w, "| Benchmark\t| Measurement\t| H-Score\t| Z-Score\t| % Change\t| Stable Reg\t| Patch Value\t|") @@ -323,8 +334,7 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { if testCount == 0 { comment.Reset() - comment.WriteString("# 👋GoDriver Performance\n") - comment.WriteString("There were no significant changes to the performance to report.") + fmt.Fprintf(&comment, "There were no significant changes to the performance to report for version %s.\n", version) } comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") From cd7c4da4361c78bc5bf71b51978bfec267448d72 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 30 Jul 2025 11:24:30 -0400 Subject: [PATCH 02/12] Generate formatted perf comment with dynamic Evg URLs --- internal/cmd/perfcomp/parseperfcomp/main.go | 122 ++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 internal/cmd/perfcomp/parseperfcomp/main.go diff --git a/internal/cmd/perfcomp/parseperfcomp/main.go b/internal/cmd/perfcomp/parseperfcomp/main.go new file mode 100644 index 0000000000..d4143463d6 --- /dev/null +++ b/internal/cmd/perfcomp/parseperfcomp/main.go @@ -0,0 +1,122 @@ +// Copyright (C) MongoDB, Inc. 2025-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package main + +import ( + "bufio" + "fmt" + "log" + "net/url" + "os" + "strings" +) + +const parsePerfCompDir = "./internal/cmd/perfcomp/parseperfcomp/" +const perfReportFileTxt = "perf-report.txt" +const perfReportFileMd = "perf-report.md" +const perfVariant = "^perf$" + +func main() { + var line string + + // open file to read + fRead, err := os.Open(parsePerfCompDir + perfReportFileTxt) + if err != nil { + log.Fatalf("Could not open %s: %v", perfReportFileTxt, err) + } + defer fRead.Close() + + // open file to write + fWrite, err := os.Create(perfReportFileMd) + if err != nil { + log.Fatalf("Could not create %s: %v", perfReportFileMd, err) + } + defer fWrite.Close() + + fmt.Fprintf(fWrite, "## 👋GoDriver Performance\n") + + // read the file line by line using scanner + scanner := bufio.NewScanner(fRead) + + var version string + var evgLink string + + for scanner.Scan() { + line = scanner.Text() + if strings.Contains(line, "Version ID:") { + // parse version + version = strings.Split(line, " ")[2] + } else if strings.Contains(line, "Commit SHA:") { + // parse commit SHA and write header + fmt.Fprintf(fWrite, "\n
\n%s\n\t
\n\n", line) + } else if strings.Contains(line, "version "+version) { + // dynamic Evergreen perf task link + evgLink, err = generateEvgLink(version, perfVariant) + if err != nil { + log.Println(err) + fmt.Fprintf(fWrite, "%s\n", line) + } else { + printUrlToLine(fWrite, line, evgLink, "version", -1) + } + } else if strings.Contains(line, "For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.") { + // last line of comment + evgLink, err = generateEvgLink(version, "") + if err != nil { + log.Println(err) + fmt.Fprintf(fWrite, "%s\n", line) + } else { + printUrlToLine(fWrite, line, evgLink, "Evergreen", 0) + } + } else { + // all other regular lines + fmt.Fprintf(fWrite, "%s\n", line) + } + } + + fmt.Fprintf(fWrite, "
\n") +} + +func generateEvgLink(version string, variant string) (string, error) { + baseUrl := "https://spruce.mongodb.com" + page := "0" + sorts := "STATUS:ASC;BASE_STATUS:DESC" + + u, err := url.Parse(baseUrl) + if err != nil { + return "", fmt.Errorf("Error parsing URL: %v", err) + } + + u.Path = fmt.Sprintf("version/%s/tasks", version) + + // construct query parameters + queryParams := url.Values{} + queryParams.Add("page", page) + queryParams.Add("sorts", sorts) + if variant != "" { + queryParams.Add("variant", variant) + } + + u.RawQuery = queryParams.Encode() + return u.String(), nil +} + +func printUrlToLine(fWrite *os.File, line string, link string, targetWord string, step int) { + words := strings.Split(line, " ") + for i, w := range words { + if i > 0 && words[i+step] == targetWord { + fmt.Fprintf(fWrite, "[%s](%s)", w, link) + } else { + fmt.Fprint(fWrite, w) + } + + if i < len(words)-1 { + fmt.Fprint(fWrite, " ") + } else { + fmt.Fprint(fWrite, "\n") + } + } +} From 50c197242d32460fbf7d59a957e47002453fbb67 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 30 Jul 2025 11:26:42 -0400 Subject: [PATCH 03/12] Make PR comment script and include permissions --- .evergreen/config.yml | 2 ++ etc/perf-pr-comment.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 67664eb5b3..857989a289 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -290,6 +290,7 @@ functions: type: test params: binary: bash + add_expansions_to_env: true env: VERSION_ID: ${version_id} BASE_SHA: "${revision}" @@ -695,6 +696,7 @@ tasks: params: binary: bash args: [*task-runner, driver-benchmark] + - func: assume-test-secrets-ec2-role - func: send-perf-data - func: send-perf-pr-comment diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index b39d24a3ec..325f0b7e01 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -13,8 +13,8 @@ if [[ -n "${BASE_SHA+set}" && -n "${HEAD_SHA+set}" && "$BASE_SHA" != "$HEAD_SHA" # Parse and generate perf comparison comment. go run ./internal/cmd/perfcomp/parseperfcomp/main.go # Make the PR comment. - # target=$DRIVERS_TOOLS/.evergreen/github_app/create_or_modify_comment.sh - # bash $target -m "## 👋GoDriver Performance" -c "$(pwd)/perf-report.md" -h $HEAD_SHA -o "mongodb" -n "mongo-go-driver" + target=$DRIVERS_TOOLS/.evergreen/github_app/create_or_modify_comment.sh + bash $target -m "## 👋GoDriver Performance" -c "$(pwd)/perf-report.md" -h $HEAD_SHA -o "mongodb" -n "mongo-go-driver" else # Skip comment if it isn't a PR run. echo "Skipping Perf PR comment" From d81091aaef9b59ca2d447c76eca66ab615783f59 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 30 Jul 2025 13:49:35 -0400 Subject: [PATCH 04/12] Nit: add space in header --- etc/perf-pr-comment.sh | 2 +- internal/cmd/perfcomp/main.go | 2 +- internal/cmd/perfcomp/parseperfcomp/main.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index 325f0b7e01..e9985b5e46 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -14,7 +14,7 @@ if [[ -n "${BASE_SHA+set}" && -n "${HEAD_SHA+set}" && "$BASE_SHA" != "$HEAD_SHA" go run ./internal/cmd/perfcomp/parseperfcomp/main.go # Make the PR comment. target=$DRIVERS_TOOLS/.evergreen/github_app/create_or_modify_comment.sh - bash $target -m "## 👋GoDriver Performance" -c "$(pwd)/perf-report.md" -h $HEAD_SHA -o "mongodb" -n "mongo-go-driver" + bash $target -m "## 👋 GoDriver Performance" -c "$(pwd)/perf-report.md" -h $HEAD_SHA -o "mongodb" -n "mongo-go-driver" else # Skip comment if it isn't a PR run. echo "Skipping Perf PR comment" diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 0424c5ff92..e6451961ea 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -169,7 +169,7 @@ func main() { // Log energy stats output prComment := generatePRComment(allEnergyStats, version) - log.Println("👋GoDriver Performance") + log.Println("👋 GoDriver Performance") log.Println(prComment) // Save for PR comment if it is a PR run diff --git a/internal/cmd/perfcomp/parseperfcomp/main.go b/internal/cmd/perfcomp/parseperfcomp/main.go index d4143463d6..20821831fb 100644 --- a/internal/cmd/perfcomp/parseperfcomp/main.go +++ b/internal/cmd/perfcomp/parseperfcomp/main.go @@ -37,7 +37,7 @@ func main() { } defer fWrite.Close() - fmt.Fprintf(fWrite, "## 👋GoDriver Performance\n") + fmt.Fprintf(fWrite, "## 👋 GoDriver Performance\n") // read the file line by line using scanner scanner := bufio.NewScanner(fRead) From 6d8358c0886c30216bd5424ef61636d3957e3bf5 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 30 Jul 2025 14:45:17 -0400 Subject: [PATCH 05/12] Sort significant benchmarks by percent change --- internal/cmd/perfcomp/main.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index e6451961ea..e5428343fe 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -16,6 +16,7 @@ import ( "log" "math" "os" + "sort" "strings" "text/tabwriter" "time" @@ -323,19 +324,25 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { fmt.Fprintln(w, "| Benchmark\t| Measurement\t| H-Score\t| Z-Score\t| % Change\t| Stable Reg\t| Patch Value\t|") fmt.Fprintln(w, "| ---------\t| -----------\t| -------\t| -------\t| --------\t| ----------\t| -----------\t|") - var testCount int64 + var significantEnergyStats []EnergyStats for _, es := range energyStats { - if math.Abs(es.ZScore) > 1.96 { - testCount += 1 - fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t|\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.MeasurementVal) + if es.Measurement != "iterations" && math.Abs(es.ZScore) > 1.96 { + significantEnergyStats = append(significantEnergyStats, *es) } } - w.Flush() - if testCount == 0 { + if len(significantEnergyStats) == 0 { comment.Reset() fmt.Fprintf(&comment, "There were no significant changes to the performance to report for version %s.\n", version) + } else { + sort.Slice(significantEnergyStats, func(i, j int) bool { + return math.Abs(significantEnergyStats[i].PercentChange) > math.Abs(significantEnergyStats[j].PercentChange) + }) + for _, es := range significantEnergyStats { + fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t|\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.MeasurementVal) + } } + w.Flush() comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") return comment.String() From bf441950aa7f0c526d44a4112753e1df4152108a Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Thu, 31 Jul 2025 12:13:40 -0400 Subject: [PATCH 06/12] Bold summary of dropdown --- internal/cmd/perfcomp/parseperfcomp/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/perfcomp/parseperfcomp/main.go b/internal/cmd/perfcomp/parseperfcomp/main.go index 20821831fb..cb47a9abd6 100644 --- a/internal/cmd/perfcomp/parseperfcomp/main.go +++ b/internal/cmd/perfcomp/parseperfcomp/main.go @@ -52,7 +52,7 @@ func main() { version = strings.Split(line, " ")[2] } else if strings.Contains(line, "Commit SHA:") { // parse commit SHA and write header - fmt.Fprintf(fWrite, "\n
\n%s\n\t
\n\n", line) + fmt.Fprintf(fWrite, "\n
\n%s\n\t
\n\n", line) } else if strings.Contains(line, "version "+version) { // dynamic Evergreen perf task link evgLink, err = generateEvgLink(version, perfVariant) From ce710ae3d00be731542655903f1f9a942f99086d Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Thu, 31 Jul 2025 18:39:59 -0400 Subject: [PATCH 07/12] Reiterate on display and include comments --- internal/cmd/perfcomp/main.go | 10 +++++++--- internal/cmd/perfcomp/parseperfcomp/main.go | 11 +++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index e5428343fe..4e60812636 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -321,11 +321,15 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n\n", version) w := tabwriter.NewWriter(&comment, 0, 0, 1, ' ', 0) - fmt.Fprintln(w, "| Benchmark\t| Measurement\t| H-Score\t| Z-Score\t| % Change\t| Stable Reg\t| Patch Value\t|") - fmt.Fprintln(w, "| ---------\t| -----------\t| -------\t| -------\t| --------\t| ----------\t| -----------\t|") + fmt.Fprintln(w, "| Benchmark\t| Measurement\t| % Change\t| Patch Value\t| Stable Region\t| H-Score\t| Z-Score\t| ") + fmt.Fprintln(w, "| ---------\t| -----------\t| --------\t| -----------\t| -------------\t| -------\t| -------\t|") var significantEnergyStats []EnergyStats for _, es := range energyStats { + // The "iterations" measurement is the number of iterations that the Go + // benchmark suite had to run to converge on a benchmark measurement. It + // is not comparable between benchmark runs, so is not a useful + // measurement to print here. Omit it. if es.Measurement != "iterations" && math.Abs(es.ZScore) > 1.96 { significantEnergyStats = append(significantEnergyStats, *es) } @@ -339,7 +343,7 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { return math.Abs(significantEnergyStats[i].PercentChange) > math.Abs(significantEnergyStats[j].PercentChange) }) for _, es := range significantEnergyStats { - fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t|\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.MeasurementVal) + fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t| %.4f\t|\n", es.Benchmark, es.Measurement, es.PercentChange, es.MeasurementVal, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.HScore, es.ZScore) } } w.Flush() diff --git a/internal/cmd/perfcomp/parseperfcomp/main.go b/internal/cmd/perfcomp/parseperfcomp/main.go index cb47a9abd6..b2cd7414e5 100644 --- a/internal/cmd/perfcomp/parseperfcomp/main.go +++ b/internal/cmd/perfcomp/parseperfcomp/main.go @@ -19,6 +19,8 @@ const parsePerfCompDir = "./internal/cmd/perfcomp/parseperfcomp/" const perfReportFileTxt = "perf-report.txt" const perfReportFileMd = "perf-report.md" const perfVariant = "^perf$" +const hscoreDefLink = "https://en.wikipedia.org/wiki/Energy_distance#:~:text=E%2Dcoefficient%20of%20inhomogeneity" +const zscoreDefLink = "https://en.wikipedia.org/wiki/Standard_score#Calculation" func main() { var line string @@ -71,6 +73,15 @@ func main() { } else { printUrlToLine(fWrite, line, evgLink, "Evergreen", 0) } + } else if strings.Contains(line, ", ") { + line = strings.ReplaceAll(line, ", ", "
") + fmt.Fprintf(fWrite, "%s\n", line) + } else if strings.Contains(line, "H-Score") { + linkedWord := "[H-Score](" + hscoreDefLink + ")" + line = strings.ReplaceAll(line, "H-Score", linkedWord) + linkedWord = "[Z-Score](" + zscoreDefLink + ")" + line = strings.ReplaceAll(line, "Z-Score", linkedWord) + fmt.Fprintf(fWrite, "%s\n", line) } else { // all other regular lines fmt.Fprintf(fWrite, "%s\n", line) From 2205f197d427a0ea2ba5ccd4c665d43fd233e418 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Fri, 1 Aug 2025 10:51:04 -0400 Subject: [PATCH 08/12] Rename header to Performance Results --- etc/perf-pr-comment.sh | 2 +- internal/cmd/perfcomp/main.go | 2 +- internal/cmd/perfcomp/parseperfcomp/main.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index e9985b5e46..4758a0138d 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -14,7 +14,7 @@ if [[ -n "${BASE_SHA+set}" && -n "${HEAD_SHA+set}" && "$BASE_SHA" != "$HEAD_SHA" go run ./internal/cmd/perfcomp/parseperfcomp/main.go # Make the PR comment. target=$DRIVERS_TOOLS/.evergreen/github_app/create_or_modify_comment.sh - bash $target -m "## 👋 GoDriver Performance" -c "$(pwd)/perf-report.md" -h $HEAD_SHA -o "mongodb" -n "mongo-go-driver" + bash $target -m "## 🧪 Performance Results" -c "$(pwd)/perf-report.md" -h $HEAD_SHA -o "mongodb" -n "mongo-go-driver" else # Skip comment if it isn't a PR run. echo "Skipping Perf PR comment" diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 4e60812636..a060285491 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -170,7 +170,7 @@ func main() { // Log energy stats output prComment := generatePRComment(allEnergyStats, version) - log.Println("👋 GoDriver Performance") + log.Println("🧪 Performance Results") log.Println(prComment) // Save for PR comment if it is a PR run diff --git a/internal/cmd/perfcomp/parseperfcomp/main.go b/internal/cmd/perfcomp/parseperfcomp/main.go index b2cd7414e5..6e163b54f5 100644 --- a/internal/cmd/perfcomp/parseperfcomp/main.go +++ b/internal/cmd/perfcomp/parseperfcomp/main.go @@ -39,7 +39,7 @@ func main() { } defer fWrite.Close() - fmt.Fprintf(fWrite, "## 👋 GoDriver Performance\n") + fmt.Fprintf(fWrite, "## 🧪 Performance Results\n") // read the file line by line using scanner scanner := bufio.NewScanner(fRead) From 9e2ca6d3099050f4f14f65a305113ede87edf524 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 4 Aug 2025 13:43:57 -0400 Subject: [PATCH 09/12] Set up and install cobra --- .gitignore | 1 + etc/perf-pr-comment.sh | 11 +++++++---- internal/cmd/perfcomp/go.mod | 6 ++++++ internal/cmd/perfcomp/go.sum | 8 ++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 11db098a16..2230de2843 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ internal/cmd/compilecheck/compilecheck.so # Ignore api report files api-report.md api-report.txt +bin # Ignore perf report files perf-report.md diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index 4758a0138d..1d40eadf53 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -4,20 +4,23 @@ set -eux -# Generate perf report. pushd ./internal/cmd/perfcomp >/dev/null || exist -GOWORK=off go run main.go --project="mongo-go-driver" ${VERSION_ID} > ./parseperfcomp/perf-report.txt +GOWORK=off go build -o ../../../bin/perfcomp . popd >/dev/null +# Generate perf report. +GOWORK=off ./bin/perfcomp compare --project="mongo-go-driver" ${VERSION_ID} > ./internal/cmd/perfcomp/perf-report.txt + if [[ -n "${BASE_SHA+set}" && -n "${HEAD_SHA+set}" && "$BASE_SHA" != "$HEAD_SHA" ]]; then # Parse and generate perf comparison comment. - go run ./internal/cmd/perfcomp/parseperfcomp/main.go + GOWORK=off ./bin/perfcomp mdreport # Make the PR comment. target=$DRIVERS_TOOLS/.evergreen/github_app/create_or_modify_comment.sh bash $target -m "## 🧪 Performance Results" -c "$(pwd)/perf-report.md" -h $HEAD_SHA -o "mongodb" -n "mongo-go-driver" + rm ./perf-report.md else # Skip comment if it isn't a PR run. echo "Skipping Perf PR comment" fi -rm ./internal/cmd/perfcomp/parseperfcomp/perf-report.txt +rm ./internal/cmd/perfcomp/perf-report.txt diff --git a/internal/cmd/perfcomp/go.mod b/internal/cmd/perfcomp/go.mod index 3b42147b1c..2a3de5fd8b 100644 --- a/internal/cmd/perfcomp/go.mod +++ b/internal/cmd/perfcomp/go.mod @@ -12,11 +12,17 @@ require ( gonum.org/v1/gonum v0.16.0 ) +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.6 // indirect +) + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/cobra v1.9.1 github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect diff --git a/internal/cmd/perfcomp/go.sum b/internal/cmd/perfcomp/go.sum index 6d55f6e933..a226fa646b 100644 --- a/internal/cmd/perfcomp/go.sum +++ b/internal/cmd/perfcomp/go.sum @@ -1,13 +1,21 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= From 8e6dd5f38a9a8372ddd92580c4cb426a319cf20e Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 4 Aug 2025 13:45:13 -0400 Subject: [PATCH 10/12] Restructure files to use cobra in main.go --- internal/cmd/perfcomp/compare.go | 480 ++++++++++++++++++ internal/cmd/perfcomp/main.go | 439 +--------------- .../{parseperfcomp/main.go => mdreport.go} | 24 +- 3 files changed, 510 insertions(+), 433 deletions(-) create mode 100644 internal/cmd/perfcomp/compare.go rename internal/cmd/perfcomp/{parseperfcomp/main.go => mdreport.go} (87%) diff --git a/internal/cmd/perfcomp/compare.go b/internal/cmd/perfcomp/compare.go new file mode 100644 index 0000000000..3bde27ea44 --- /dev/null +++ b/internal/cmd/perfcomp/compare.go @@ -0,0 +1,480 @@ +// Copyright (C) MongoDB, Inc. 2025-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +// This module cannot be included in the workspace since it requires a version of Gonum that is not compatible with the Go Driver. +// Must use GOWORK=off to run this test. + +package main + +import ( + "context" + "fmt" + "log" + "math" + "os" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" + "gonum.org/v1/gonum/mat" +) + +type OverrideInfo struct { + OverrideMainline bool `bson:"override_mainline"` + BaseOrder any `bson:"base_order"` + Reason any `bson:"reason"` + User any `bson:"user"` +} + +type Info struct { + Project string `bson:"project"` + Version string `bson:"version"` + Variant string `bson:"variant"` + Order int64 `bson:"order"` + TaskName string `bson:"task_name"` + TaskID string `bson:"task_id"` + Execution int64 `bson:"execution"` + Mainline bool `bson:"mainline"` + OverrideInfo OverrideInfo + TestName string `bson:"test_name"` + Args map[string]any `bson:"args"` +} + +type Stat struct { + Name string `bson:"name"` + Val float64 `bson:"val"` + Metadata any `bson:"metadata"` +} + +type Rollups struct { + Stats []Stat +} + +type RawData struct { + Info Info + CreatedAt any `bson:"created_at"` + CompletedAt any `bson:"completed_at"` + Rollups Rollups + FailedRollupAttempts int64 `bson:"failed_rollup_attempts"` +} + +type TimeSeriesInfo struct { + Project string `bson:"project"` + Variant string `bson:"variant"` + Task string `bson:"task"` + Test string `bson:"test"` + Measurement string `bson:"measurement"` + Args map[string]any `bson:"args"` +} + +type StableRegion struct { + TimeSeriesInfo TimeSeriesInfo + Start any `bson:"start"` + End any `bson:"end"` + Values []float64 `bson:"values"` + StartOrder int64 `bson:"start_order"` + EndOrder int64 `bson:"end_order"` + Mean float64 `bson:"mean"` + Std float64 `bson:"std"` + Median float64 `bson:"median"` + Max float64 `bson:"max"` + Min float64 `bson:"min"` + CoefficientOfVariation float64 `bson:"coefficient_of_variation"` + LastSuccessfulUpdate any `bson:"last_successful_update"` + Last bool `bson:"last"` + Contexts []any `bson:"contexts"` +} + +type EnergyStats struct { + Project string + Benchmark string + Measurement string + PatchVersion string + StableRegion StableRegion + MeasurementVal float64 + PercentChange float64 + EnergyStatistic float64 + TestStatistic float64 + HScore float64 + ZScore float64 +} + +const expandedMetricsDB = "expanded_metrics" +const rawResultsColl = "raw_results" +const stableRegionsColl = "stable_regions" + +func newCompareCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "compare", + Short: "compare evergreen patch to mainline commit", + // Version id is a required argument + Args: func(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return fmt.Errorf("this command requires an evergreen patch version ID") + } + return nil + }, + } + + // TODO (GODRIVER-3102): Map each project to a unique performance context, + // necessary for project switching to work since it's required for querying the stable region. + cmd.Flags().String("project", "mongo-go-driver", "specify the name of an existing Evergreen project") + + cmd.Run = func(cmd *cobra.Command, args []string) { + // Check for variables + uri := os.Getenv("PERF_URI_PRIVATE_ENDPOINT") + if uri == "" { + log.Fatal("PERF_URI_PRIVATE_ENDPOINT env variable is not set") + } + + // Retrieve the project flag value + project, err := cmd.Flags().GetString("project") + if err != nil { + log.Fatalf("failed to get project flag: %v", err) + } + + // Validate the project flag + if project == "" { + log.Fatal("must provide project") + } + + if err := runCompare(cmd, args, project); err != nil { + log.Fatalf("failed to compare: %v", err) + } + } + + return cmd +} + +func runCompare(cmd *cobra.Command, args []string, project string) error { + + uri := os.Getenv("PERF_URI_PRIVATE_ENDPOINT") + version := args[len(args)-1] + + // Connect to analytics node + client, err := mongo.Connect(options.Client().ApplyURI(uri)) + if err != nil { + return fmt.Errorf("Error connecting client: %v", err) + } + + defer func() { // Defer disconnect client + err = client.Disconnect(context.Background()) + if err != nil { + log.Fatalf("Failed to disconnect client: %v", err) + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err = client.Ping(ctx, nil) + if err != nil { + return fmt.Errorf("Error pinging MongoDB Analytics: %v", err) + } + log.Println("Successfully connected to MongoDB Analytics node.") + + db := client.Database(expandedMetricsDB) + + // Get raw data, most recent stable region, and calculate energy stats + findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + patchRawData, err := findRawData(findCtx, project, version, db.Collection(rawResultsColl)) + if err != nil { + return fmt.Errorf("Error getting raw data: %v", err) + } + + allEnergyStats, err := getEnergyStatsForAllBenchMarks(findCtx, patchRawData, db.Collection(stableRegionsColl)) + if err != nil { + return fmt.Errorf("Error getting energy statistics: %v", err) + } + + // Log energy stats output + prComment := generatePRComment(allEnergyStats, version) + log.Println("🧪 Performance Results") + log.Println(prComment) + + // Save for PR comment if it is a PR run + commitSHA := os.Getenv("HEAD_SHA") + if commitSHA != "" { + fmt.Printf("Version ID: %s\n", version) + fmt.Printf("Commit SHA: %s\n", commitSHA) // Use fmt to print to stdout + fmt.Println(prComment) + } + + return nil +} + +func findRawData(ctx context.Context, project string, version string, coll *mongo.Collection) ([]RawData, error) { + filter := bson.D{ + {"info.project", project}, + {"info.version", version}, + {"info.variant", "perf"}, + {"info.task_name", "perf"}, + } + + cursor, err := coll.Find(ctx, filter) + if err != nil { + log.Fatalf( + "Error retrieving raw data for version %q: %v", + version, + err, + ) + } + defer func() { + err = cursor.Close(ctx) + if err != nil { + log.Fatalf("Error closing cursor while retrieving raw data for version %q: %v", version, err) + } + }() + + log.Printf("Successfully retrieved %d docs from version %s.\n", cursor.RemainingBatchLength(), version) + + var rawData []RawData + err = cursor.All(ctx, &rawData) + if err != nil { + log.Fatalf( + "Error decoding raw data from version %q: %v", + version, + err, + ) + } + + return rawData, err +} + +// Find the most recent stable region of the mainline version for a specific test/measurement +func findLastStableRegion(ctx context.Context, project string, testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { + filter := bson.D{ + {"time_series_info.project", project}, + {"time_series_info.variant", "perf"}, + {"time_series_info.task", "perf"}, + {"time_series_info.test", testname}, + {"time_series_info.measurement", measurement}, + {"last", true}, + {"contexts", bson.D{{"$in", bson.A{"GoDriver perf task"}}}}, // TODO (GODRIVER-3102): Refactor perf context for project switching. + } + + findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}) + + var sr *StableRegion + err := coll.FindOne(ctx, filter, findOptions).Decode(&sr) + if err != nil { + return nil, err + } + return sr, nil +} + +// For a specific test and measurement +func getEnergyStatsForOneBenchmark(ctx context.Context, rd RawData, coll *mongo.Collection) ([]*EnergyStats, error) { + testname := rd.Info.TestName + var energyStats []*EnergyStats + + for i := range rd.Rollups.Stats { + project := rd.Info.Project + measName := rd.Rollups.Stats[i].Name + measVal := rd.Rollups.Stats[i].Val + + stableRegion, err := findLastStableRegion(ctx, project, testname, measName, coll) + if err != nil { + log.Fatalf( + "Error finding last stable region for test %q, measurement %q: %v", + testname, + measName, + err, + ) + } + + // The performance analyzer compares the measurement value from the patch to a stable region that succeeds the latest change point. + // For example, if there were 5 measurements since the last change point, then the stable region is the 5 latest values for the measurement. + stableRegionVec := mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values) + measValVec := mat.NewDense(1, 1, []float64{measVal}) // singleton + + estat, tstat, hscore, err := getEnergyStatistics(stableRegionVec, measValVec) + if err != nil { + log.Fatalf( + "Could not calculate energy stats for test %q, measurement %q: %v", + testname, + measName, + err, + ) + } + + zscore := getZScore(measVal, stableRegion.Mean, stableRegion.Std) + pChange := getPercentageChange(measVal, stableRegion.Mean) + + es := EnergyStats{ + Project: project, + Benchmark: testname, + Measurement: measName, + PatchVersion: rd.Info.Version, + StableRegion: *stableRegion, + MeasurementVal: measVal, + PercentChange: pChange, + EnergyStatistic: estat, + TestStatistic: tstat, + HScore: hscore, + ZScore: zscore, + } + energyStats = append(energyStats, &es) + } + + return energyStats, nil +} + +func getEnergyStatsForAllBenchMarks(ctx context.Context, patchRawData []RawData, coll *mongo.Collection) ([]*EnergyStats, error) { + var allEnergyStats []*EnergyStats + for _, rd := range patchRawData { + energyStats, err := getEnergyStatsForOneBenchmark(ctx, rd, coll) + if err != nil { + log.Fatalf( + "Could not get energy stats for %q: %v", + rd.Info.TestName, + err, + ) + } else { + allEnergyStats = append(allEnergyStats, energyStats...) + } + } + return allEnergyStats, nil +} + +func generatePRComment(energyStats []*EnergyStats, version string) string { + var comment strings.Builder + fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n\n", version) + + w := tabwriter.NewWriter(&comment, 0, 0, 1, ' ', 0) + fmt.Fprintln(w, "| Benchmark\t| Measurement\t| % Change\t| Patch Value\t| Stable Region\t| H-Score\t| Z-Score\t| ") + fmt.Fprintln(w, "| ---------\t| -----------\t| --------\t| -----------\t| -------------\t| -------\t| -------\t|") + + var significantEnergyStats []EnergyStats + for _, es := range energyStats { + // The "iterations" measurement is the number of iterations that the Go + // benchmark suite had to run to converge on a benchmark measurement. It + // is not comparable between benchmark runs, so is not a useful + // measurement to print here. Omit it. + if es.Measurement != "iterations" && math.Abs(es.ZScore) > 1.96 { + significantEnergyStats = append(significantEnergyStats, *es) + } + } + + if len(significantEnergyStats) == 0 { + comment.Reset() + fmt.Fprintf(&comment, "There were no significant changes to the performance to report for version %s.\n", version) + } else { + sort.Slice(significantEnergyStats, func(i, j int) bool { + return math.Abs(significantEnergyStats[i].PercentChange) > math.Abs(significantEnergyStats[j].PercentChange) + }) + for _, es := range significantEnergyStats { + fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t| %.4f\t|\n", es.Benchmark, es.Measurement, es.PercentChange, es.MeasurementVal, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.HScore, es.ZScore) + } + } + w.Flush() + + comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") + return comment.String() +} + +// Given two matrices, this function returns +// (e, t, h) = (E-statistic, test statistic, e-coefficient of inhomogeneity) +func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64, error) { + xrows, xcols := x.Dims() + yrows, ycols := y.Dims() + + if xcols != ycols { + return 0, 0, 0, fmt.Errorf("both inputs must have the same number of columns") + } + if xrows == 0 || yrows == 0 { + return 0, 0, 0, fmt.Errorf("inputs cannot be empty") + } + + xrowsf := float64(xrows) + yrowsf := float64(yrows) + + var A float64 // E|X-Y| + if xrowsf > 0 && yrowsf > 0 { + dist, err := getDistance(x, y) + if err != nil { + return 0, 0, 0, err + } + A = dist / (xrowsf * yrowsf) + } else { + A = 0 + } + + var B float64 // E|X-X'| + if xrowsf > 0 { + dist, err := getDistance(x, x) + if err != nil { + return 0, 0, 0, err + } + B = dist / (xrowsf * xrowsf) + } else { + B = 0 + } + + var C float64 // E|Y-Y'| + if yrowsf > 0 { + dist, err := getDistance(y, y) + if err != nil { + return 0, 0, 0, err + } + C = dist / (yrowsf * yrowsf) + } else { + C = 0 + } + + E := 2*A - B - C // D^2(F_x, F_y) + T := ((xrowsf * yrowsf) / (xrowsf + yrowsf)) * E + var H float64 + if A > 0 { + H = E / (2 * A) + } else { + H = 0 + } + return E, T, H, nil +} + +// Given two vectors (expected 1 col), +// this function returns the sum of distances between each pair. +func getDistance(x, y *mat.Dense) (float64, error) { + xrows, xcols := x.Dims() + yrows, ycols := y.Dims() + + if xcols != 1 || ycols != 1 { + return 0, fmt.Errorf("both inputs must be column vectors") + } + + var sum float64 + + for i := 0; i < xrows; i++ { + for j := 0; j < yrows; j++ { + sum += math.Abs(x.At(i, 0) - y.At(j, 0)) + } + } + return sum, nil +} + +// Get Z score for result x, compared to mean u and st dev o. +func getZScore(x, mu, sigma float64) float64 { + if sigma == 0 { + return math.NaN() + } + return (x - mu) / sigma +} + +// Get percentage change for result x compared to mean u. +func getPercentageChange(x, mu float64) float64 { + if mu == 0 { + return math.NaN() + } + return ((x - mu) / mu) * 100 +} diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index a060285491..c6d921f390 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -4,446 +4,25 @@ // not use this file except in compliance with the License. You may obtain // a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -// This module cannot be included in the workspace since it requires a version of Gonum that is not compatible with the Go Driver. -// Must use GOWORK=off to run this test. - package main import ( - "context" - "flag" - "fmt" "log" - "math" - "os" - "sort" - "strings" - "text/tabwriter" - "time" - "go.mongodb.org/mongo-driver/v2/bson" - "go.mongodb.org/mongo-driver/v2/mongo" - "go.mongodb.org/mongo-driver/v2/mongo/options" - "gonum.org/v1/gonum/mat" + "github.com/spf13/cobra" ) -type OverrideInfo struct { - OverrideMainline bool `bson:"override_mainline"` - BaseOrder any `bson:"base_order"` - Reason any `bson:"reason"` - User any `bson:"user"` -} - -type Info struct { - Project string `bson:"project"` - Version string `bson:"version"` - Variant string `bson:"variant"` - Order int64 `bson:"order"` - TaskName string `bson:"task_name"` - TaskID string `bson:"task_id"` - Execution int64 `bson:"execution"` - Mainline bool `bson:"mainline"` - OverrideInfo OverrideInfo - TestName string `bson:"test_name"` - Args map[string]any `bson:"args"` -} - -type Stat struct { - Name string `bson:"name"` - Val float64 `bson:"val"` - Metadata any `bson:"metadata"` -} - -type Rollups struct { - Stats []Stat -} - -type RawData struct { - Info Info - CreatedAt any `bson:"created_at"` - CompletedAt any `bson:"completed_at"` - Rollups Rollups - FailedRollupAttempts int64 `bson:"failed_rollup_attempts"` -} - -type TimeSeriesInfo struct { - Project string `bson:"project"` - Variant string `bson:"variant"` - Task string `bson:"task"` - Test string `bson:"test"` - Measurement string `bson:"measurement"` - Args map[string]any `bson:"args"` -} - -type StableRegion struct { - TimeSeriesInfo TimeSeriesInfo - Start any `bson:"start"` - End any `bson:"end"` - Values []float64 `bson:"values"` - StartOrder int64 `bson:"start_order"` - EndOrder int64 `bson:"end_order"` - Mean float64 `bson:"mean"` - Std float64 `bson:"std"` - Median float64 `bson:"median"` - Max float64 `bson:"max"` - Min float64 `bson:"min"` - CoefficientOfVariation float64 `bson:"coefficient_of_variation"` - LastSuccessfulUpdate any `bson:"last_successful_update"` - Last bool `bson:"last"` - Contexts []any `bson:"contexts"` -} - -type EnergyStats struct { - Project string - Benchmark string - Measurement string - PatchVersion string - StableRegion StableRegion - MeasurementVal float64 - PercentChange float64 - EnergyStatistic float64 - TestStatistic float64 - HScore float64 - ZScore float64 -} - -const expandedMetricsDB = "expanded_metrics" -const rawResultsColl = "raw_results" -const stableRegionsColl = "stable_regions" - func main() { - // Check for variables - uri := os.Getenv("PERF_URI_PRIVATE_ENDPOINT") - if uri == "" { - log.Fatal("PERF_URI_PRIVATE_ENDPOINT env variable is not set") - } - - version := os.Args[len(os.Args)-1] - if version == "" { - log.Fatal("could not get VERSION_ID") - } - - // TODO (GODRIVER-3102): Map each project to a unique performance context, - // necessary for project switching to work since it's required for querying the stable region. - project := flag.String("project", "mongo-go-driver", "specify the name of an existing Evergreen project") - if project == nil { - log.Fatalf("must provide project") - } - - // Connect to analytics node - client, err := mongo.Connect(options.Client().ApplyURI(uri)) - if err != nil { - log.Fatalf("Error connecting client: %v", err) - } - - defer func() { // Defer disconnect client - err = client.Disconnect(context.Background()) - if err != nil { - log.Fatalf("Failed to disconnect client: %v", err) - } - }() - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - err = client.Ping(ctx, nil) - if err != nil { - log.Fatalf("Error pinging MongoDB Analytics: %v", err) - } - log.Println("Successfully connected to MongoDB Analytics node.") - - db := client.Database(expandedMetricsDB) - - // Get raw data, most recent stable region, and calculate energy stats - findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - patchRawData, err := findRawData(findCtx, *project, version, db.Collection(rawResultsColl)) - if err != nil { - log.Fatalf("Error getting raw data: %v", err) - } - - allEnergyStats, err := getEnergyStatsForAllBenchMarks(findCtx, patchRawData, db.Collection(stableRegionsColl)) - if err != nil { - log.Fatalf("Error getting energy statistics: %v", err) - } - - // Log energy stats output - prComment := generatePRComment(allEnergyStats, version) - log.Println("🧪 Performance Results") - log.Println(prComment) - - // Save for PR comment if it is a PR run - commitSHA := os.Getenv("HEAD_SHA") - if commitSHA != "" { - fmt.Printf("Version ID: %s\n", version) - fmt.Printf("Commit SHA: %s\n", commitSHA) // Use fmt to print to stdout - fmt.Println(prComment) - } -} - -func findRawData(ctx context.Context, project string, version string, coll *mongo.Collection) ([]RawData, error) { - filter := bson.D{ - {"info.project", project}, - {"info.version", version}, - {"info.variant", "perf"}, - {"info.task_name", "perf"}, - } - - cursor, err := coll.Find(ctx, filter) - if err != nil { - log.Fatalf( - "Error retrieving raw data for version %q: %v", - version, - err, - ) - } - defer func() { - err = cursor.Close(ctx) - if err != nil { - log.Fatalf("Error closing cursor while retrieving raw data for version %q: %v", version, err) - } - }() - - log.Printf("Successfully retrieved %d docs from version %s.\n", cursor.RemainingBatchLength(), version) - - var rawData []RawData - err = cursor.All(ctx, &rawData) - if err != nil { - log.Fatalf( - "Error decoding raw data from version %q: %v", - version, - err, - ) - } - - return rawData, err -} - -// Find the most recent stable region of the mainline version for a specific test/measurement -func findLastStableRegion(ctx context.Context, project string, testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { - filter := bson.D{ - {"time_series_info.project", project}, - {"time_series_info.variant", "perf"}, - {"time_series_info.task", "perf"}, - {"time_series_info.test", testname}, - {"time_series_info.measurement", measurement}, - {"last", true}, - {"contexts", bson.D{{"$in", bson.A{"GoDriver perf task"}}}}, // TODO (GODRIVER-3102): Refactor perf context for project switching. + cmd := &cobra.Command{ + Use: "perfcomp", + Short: "perfcomp is a cli that reports stat-sig results between evergreen patches with the mainline commit", + Version: "0.0.0-alpha", } - findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}) - - var sr *StableRegion - err := coll.FindOne(ctx, filter, findOptions).Decode(&sr) - if err != nil { - return nil, err - } - return sr, nil -} - -// For a specific test and measurement -func getEnergyStatsForOneBenchmark(ctx context.Context, rd RawData, coll *mongo.Collection) ([]*EnergyStats, error) { - testname := rd.Info.TestName - var energyStats []*EnergyStats - - for i := range rd.Rollups.Stats { - project := rd.Info.Project - measName := rd.Rollups.Stats[i].Name - measVal := rd.Rollups.Stats[i].Val - - stableRegion, err := findLastStableRegion(ctx, project, testname, measName, coll) - if err != nil { - log.Fatalf( - "Error finding last stable region for test %q, measurement %q: %v", - testname, - measName, - err, - ) - } - - // The performance analyzer compares the measurement value from the patch to a stable region that succeeds the latest change point. - // For example, if there were 5 measurements since the last change point, then the stable region is the 5 latest values for the measurement. - stableRegionVec := mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values) - measValVec := mat.NewDense(1, 1, []float64{measVal}) // singleton - - estat, tstat, hscore, err := getEnergyStatistics(stableRegionVec, measValVec) - if err != nil { - log.Fatalf( - "Could not calculate energy stats for test %q, measurement %q: %v", - testname, - measName, - err, - ) - } - - zscore := getZScore(measVal, stableRegion.Mean, stableRegion.Std) - pChange := getPercentageChange(measVal, stableRegion.Mean) - - es := EnergyStats{ - Project: project, - Benchmark: testname, - Measurement: measName, - PatchVersion: rd.Info.Version, - StableRegion: *stableRegion, - MeasurementVal: measVal, - PercentChange: pChange, - EnergyStatistic: estat, - TestStatistic: tstat, - HScore: hscore, - ZScore: zscore, - } - energyStats = append(energyStats, &es) - } - - return energyStats, nil -} - -func getEnergyStatsForAllBenchMarks(ctx context.Context, patchRawData []RawData, coll *mongo.Collection) ([]*EnergyStats, error) { - var allEnergyStats []*EnergyStats - for _, rd := range patchRawData { - energyStats, err := getEnergyStatsForOneBenchmark(ctx, rd, coll) - if err != nil { - log.Fatalf( - "Could not get energy stats for %q: %v", - rd.Info.TestName, - err, - ) - } else { - allEnergyStats = append(allEnergyStats, energyStats...) - } - } - return allEnergyStats, nil -} - -func generatePRComment(energyStats []*EnergyStats, version string) string { - var comment strings.Builder - fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n\n", version) - - w := tabwriter.NewWriter(&comment, 0, 0, 1, ' ', 0) - fmt.Fprintln(w, "| Benchmark\t| Measurement\t| % Change\t| Patch Value\t| Stable Region\t| H-Score\t| Z-Score\t| ") - fmt.Fprintln(w, "| ---------\t| -----------\t| --------\t| -----------\t| -------------\t| -------\t| -------\t|") - - var significantEnergyStats []EnergyStats - for _, es := range energyStats { - // The "iterations" measurement is the number of iterations that the Go - // benchmark suite had to run to converge on a benchmark measurement. It - // is not comparable between benchmark runs, so is not a useful - // measurement to print here. Omit it. - if es.Measurement != "iterations" && math.Abs(es.ZScore) > 1.96 { - significantEnergyStats = append(significantEnergyStats, *es) - } - } - - if len(significantEnergyStats) == 0 { - comment.Reset() - fmt.Fprintf(&comment, "There were no significant changes to the performance to report for version %s.\n", version) - } else { - sort.Slice(significantEnergyStats, func(i, j int) bool { - return math.Abs(significantEnergyStats[i].PercentChange) > math.Abs(significantEnergyStats[j].PercentChange) - }) - for _, es := range significantEnergyStats { - fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t| %.4f\t|\n", es.Benchmark, es.Measurement, es.PercentChange, es.MeasurementVal, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.HScore, es.ZScore) - } - } - w.Flush() - - comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") - return comment.String() -} - -// Given two matrices, this function returns -// (e, t, h) = (E-statistic, test statistic, e-coefficient of inhomogeneity) -func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64, error) { - xrows, xcols := x.Dims() - yrows, ycols := y.Dims() - - if xcols != ycols { - return 0, 0, 0, fmt.Errorf("both inputs must have the same number of columns") - } - if xrows == 0 || yrows == 0 { - return 0, 0, 0, fmt.Errorf("inputs cannot be empty") - } - - xrowsf := float64(xrows) - yrowsf := float64(yrows) - - var A float64 // E|X-Y| - if xrowsf > 0 && yrowsf > 0 { - dist, err := getDistance(x, y) - if err != nil { - return 0, 0, 0, err - } - A = dist / (xrowsf * yrowsf) - } else { - A = 0 - } - - var B float64 // E|X-X'| - if xrowsf > 0 { - dist, err := getDistance(x, x) - if err != nil { - return 0, 0, 0, err - } - B = dist / (xrowsf * xrowsf) - } else { - B = 0 - } - - var C float64 // E|Y-Y'| - if yrowsf > 0 { - dist, err := getDistance(y, y) - if err != nil { - return 0, 0, 0, err - } - C = dist / (yrowsf * yrowsf) - } else { - C = 0 - } - - E := 2*A - B - C // D^2(F_x, F_y) - T := ((xrowsf * yrowsf) / (xrowsf + yrowsf)) * E - var H float64 - if A > 0 { - H = E / (2 * A) - } else { - H = 0 - } - return E, T, H, nil -} - -// Given two vectors (expected 1 col), -// this function returns the sum of distances between each pair. -func getDistance(x, y *mat.Dense) (float64, error) { - xrows, xcols := x.Dims() - yrows, ycols := y.Dims() - - if xcols != 1 || ycols != 1 { - return 0, fmt.Errorf("both inputs must be column vectors") - } - - var sum float64 - - for i := 0; i < xrows; i++ { - for j := 0; j < yrows; j++ { - sum += math.Abs(x.At(i, 0) - y.At(j, 0)) - } - } - return sum, nil -} - -// Get Z score for result x, compared to mean u and st dev o. -func getZScore(x, mu, sigma float64) float64 { - if sigma == 0 { - return math.NaN() - } - return (x - mu) / sigma -} + cmd.AddCommand(newCompareCommand()) + cmd.AddCommand(newMdCommand()) -// Get percentage change for result x compared to mean u. -func getPercentageChange(x, mu float64) float64 { - if mu == 0 { - return math.NaN() + if err := cmd.Execute(); err != nil { + log.Fatalf("error: %v", err) } - return ((x - mu) / mu) * 100 } diff --git a/internal/cmd/perfcomp/parseperfcomp/main.go b/internal/cmd/perfcomp/mdreport.go similarity index 87% rename from internal/cmd/perfcomp/parseperfcomp/main.go rename to internal/cmd/perfcomp/mdreport.go index 6e163b54f5..b9611397d1 100644 --- a/internal/cmd/perfcomp/parseperfcomp/main.go +++ b/internal/cmd/perfcomp/mdreport.go @@ -13,20 +13,37 @@ import ( "net/url" "os" "strings" + + "github.com/spf13/cobra" ) -const parsePerfCompDir = "./internal/cmd/perfcomp/parseperfcomp/" +const perfCompDir = "./internal/cmd/perfcomp/" const perfReportFileTxt = "perf-report.txt" const perfReportFileMd = "perf-report.md" const perfVariant = "^perf$" const hscoreDefLink = "https://en.wikipedia.org/wiki/Energy_distance#:~:text=E%2Dcoefficient%20of%20inhomogeneity" const zscoreDefLink = "https://en.wikipedia.org/wiki/Standard_score#Calculation" -func main() { +func newMdCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "mdreport", + Short: "generates markdown output after run", + } + + cmd.Run = func(cmd *cobra.Command, args []string) { + if err := runMdCommand(cmd, args); err != nil { + log.Fatalf("failed to generate md: %v", err) + } + } + + return cmd +} + +func runMdCommand(cmd *cobra.Command, args []string) error { var line string // open file to read - fRead, err := os.Open(parsePerfCompDir + perfReportFileTxt) + fRead, err := os.Open(perfCompDir + perfReportFileTxt) if err != nil { log.Fatalf("Could not open %s: %v", perfReportFileTxt, err) } @@ -89,6 +106,7 @@ func main() { } fmt.Fprintf(fWrite, "
\n") + return nil } func generateEvgLink(version string, variant string) (string, error) { From d8c6c145fdbb3e21cc8a0315db7456bacb68e7be Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 4 Aug 2025 13:46:46 -0400 Subject: [PATCH 11/12] Add bin in gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2230de2843..c4a9fcbfad 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ go.work.sum .task env.sh expansion.yml +bin # AWS SAM-generated files internal/cmd/faas/awslambda/.aws-sam @@ -37,7 +38,6 @@ internal/cmd/compilecheck/compilecheck.so # Ignore api report files api-report.md api-report.txt -bin # Ignore perf report files perf-report.md From 7c5791bebc63638703b19f24daa3b0043ad9cd7c Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 4 Aug 2025 14:01:27 -0400 Subject: [PATCH 12/12] Add check for correct project flag --- internal/cmd/perfcomp/compare.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/cmd/perfcomp/compare.go b/internal/cmd/perfcomp/compare.go index 3bde27ea44..a03fb2e950 100644 --- a/internal/cmd/perfcomp/compare.go +++ b/internal/cmd/perfcomp/compare.go @@ -144,6 +144,8 @@ func newCompareCommand() *cobra.Command { // Validate the project flag if project == "" { log.Fatal("must provide project") + } else if project != "mongo-go-driver" { + log.Fatalf("support for project %q is not configured yet", project) } if err := runCompare(cmd, args, project); err != nil {