diff --git a/.evergreen/setup-system.sh b/.evergreen/setup-system.sh index 542060fee4..a8eb58698f 100755 --- a/.evergreen/setup-system.sh +++ b/.evergreen/setup-system.sh @@ -47,7 +47,8 @@ fi # Ensure a checkout of drivers-tools. if [ ! -d "$DRIVERS_TOOLS" ]; then - git clone https://github.com/mongodb-labs/drivers-evergreen-tools $DRIVERS_TOOLS + #git clone https://github.com/mongodb-labs/drivers-evergreen-tools $DRIVERS_TOOLS + git clone -b perfcomp https://github.com/zhouselena/drivers-evergreen-tools.git $DRIVERS_TOOLS fi # Write the .env file for drivers-tools. diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index 1d40eadf53..54cbb7aaee 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -4,23 +4,22 @@ set -eux -pushd ./internal/cmd/perfcomp >/dev/null || exist -GOWORK=off go build -o ../../../bin/perfcomp . -popd >/dev/null +pushd $DRIVERS_TOOLS/.evergreen >/dev/null +sh run-perf-comp.sh -# Generate perf report. -GOWORK=off ./bin/perfcomp compare --project="mongo-go-driver" ${VERSION_ID} > ./internal/cmd/perfcomp/perf-report.txt +./perfcomp/bin/perfcomp compare --project="mongo-go-driver" ${VERSION_ID} -if [[ -n "${BASE_SHA+set}" && -n "${HEAD_SHA+set}" && "$BASE_SHA" != "$HEAD_SHA" ]]; then + if [[ -n "${BASE_SHA+set}" && -n "${HEAD_SHA+set}" && "$BASE_SHA" != "$HEAD_SHA" ]]; then # Parse and generate perf comparison comment. - GOWORK=off ./bin/perfcomp mdreport + GOWORK=off ./perfcomp/bin/perfcomp mdreport # Make the PR comment. - target=$DRIVERS_TOOLS/.evergreen/github_app/create_or_modify_comment.sh + target=github_app/create_or_modify_comment.sh bash $target -m "## 🧪 Performance Results" -c "$(pwd)/perf-report.md" -h $HEAD_SHA -o "mongodb" -n "mongo-go-driver" + rm ./perf-report.txt rm ./perf-report.md else # Skip comment if it isn't a PR run. echo "Skipping Perf PR comment" fi -rm ./internal/cmd/perfcomp/perf-report.txt +popd >/dev/null diff --git a/internal/cmd/perfcomp/compare.go b/internal/cmd/perfcomp/compare.go deleted file mode 100644 index a03fb2e950..0000000000 --- a/internal/cmd/perfcomp/compare.go +++ /dev/null @@ -1,482 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2025-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// This module cannot be included in the workspace since it requires a version of Gonum that is not compatible with the Go Driver. -// Must use GOWORK=off to run this test. - -package main - -import ( - "context" - "fmt" - "log" - "math" - "os" - "sort" - "strings" - "text/tabwriter" - "time" - - "github.com/spf13/cobra" - "go.mongodb.org/mongo-driver/v2/bson" - "go.mongodb.org/mongo-driver/v2/mongo" - "go.mongodb.org/mongo-driver/v2/mongo/options" - "gonum.org/v1/gonum/mat" -) - -type OverrideInfo struct { - OverrideMainline bool `bson:"override_mainline"` - BaseOrder any `bson:"base_order"` - Reason any `bson:"reason"` - User any `bson:"user"` -} - -type Info struct { - Project string `bson:"project"` - Version string `bson:"version"` - Variant string `bson:"variant"` - Order int64 `bson:"order"` - TaskName string `bson:"task_name"` - TaskID string `bson:"task_id"` - Execution int64 `bson:"execution"` - Mainline bool `bson:"mainline"` - OverrideInfo OverrideInfo - TestName string `bson:"test_name"` - Args map[string]any `bson:"args"` -} - -type Stat struct { - Name string `bson:"name"` - Val float64 `bson:"val"` - Metadata any `bson:"metadata"` -} - -type Rollups struct { - Stats []Stat -} - -type RawData struct { - Info Info - CreatedAt any `bson:"created_at"` - CompletedAt any `bson:"completed_at"` - Rollups Rollups - FailedRollupAttempts int64 `bson:"failed_rollup_attempts"` -} - -type TimeSeriesInfo struct { - Project string `bson:"project"` - Variant string `bson:"variant"` - Task string `bson:"task"` - Test string `bson:"test"` - Measurement string `bson:"measurement"` - Args map[string]any `bson:"args"` -} - -type StableRegion struct { - TimeSeriesInfo TimeSeriesInfo - Start any `bson:"start"` - End any `bson:"end"` - Values []float64 `bson:"values"` - StartOrder int64 `bson:"start_order"` - EndOrder int64 `bson:"end_order"` - Mean float64 `bson:"mean"` - Std float64 `bson:"std"` - Median float64 `bson:"median"` - Max float64 `bson:"max"` - Min float64 `bson:"min"` - CoefficientOfVariation float64 `bson:"coefficient_of_variation"` - LastSuccessfulUpdate any `bson:"last_successful_update"` - Last bool `bson:"last"` - Contexts []any `bson:"contexts"` -} - -type EnergyStats struct { - Project string - Benchmark string - Measurement string - PatchVersion string - StableRegion StableRegion - MeasurementVal float64 - PercentChange float64 - EnergyStatistic float64 - TestStatistic float64 - HScore float64 - ZScore float64 -} - -const expandedMetricsDB = "expanded_metrics" -const rawResultsColl = "raw_results" -const stableRegionsColl = "stable_regions" - -func newCompareCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "compare", - Short: "compare evergreen patch to mainline commit", - // Version id is a required argument - Args: func(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return fmt.Errorf("this command requires an evergreen patch version ID") - } - return nil - }, - } - - // TODO (GODRIVER-3102): Map each project to a unique performance context, - // necessary for project switching to work since it's required for querying the stable region. - cmd.Flags().String("project", "mongo-go-driver", "specify the name of an existing Evergreen project") - - cmd.Run = func(cmd *cobra.Command, args []string) { - // Check for variables - uri := os.Getenv("PERF_URI_PRIVATE_ENDPOINT") - if uri == "" { - log.Fatal("PERF_URI_PRIVATE_ENDPOINT env variable is not set") - } - - // Retrieve the project flag value - project, err := cmd.Flags().GetString("project") - if err != nil { - log.Fatalf("failed to get project flag: %v", err) - } - - // Validate the project flag - if project == "" { - log.Fatal("must provide project") - } else if project != "mongo-go-driver" { - log.Fatalf("support for project %q is not configured yet", project) - } - - if err := runCompare(cmd, args, project); err != nil { - log.Fatalf("failed to compare: %v", err) - } - } - - return cmd -} - -func runCompare(cmd *cobra.Command, args []string, project string) error { - - uri := os.Getenv("PERF_URI_PRIVATE_ENDPOINT") - version := args[len(args)-1] - - // Connect to analytics node - client, err := mongo.Connect(options.Client().ApplyURI(uri)) - if err != nil { - return fmt.Errorf("Error connecting client: %v", err) - } - - defer func() { // Defer disconnect client - err = client.Disconnect(context.Background()) - if err != nil { - log.Fatalf("Failed to disconnect client: %v", err) - } - }() - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - err = client.Ping(ctx, nil) - if err != nil { - return fmt.Errorf("Error pinging MongoDB Analytics: %v", err) - } - log.Println("Successfully connected to MongoDB Analytics node.") - - db := client.Database(expandedMetricsDB) - - // Get raw data, most recent stable region, and calculate energy stats - findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - patchRawData, err := findRawData(findCtx, project, version, db.Collection(rawResultsColl)) - if err != nil { - return fmt.Errorf("Error getting raw data: %v", err) - } - - allEnergyStats, err := getEnergyStatsForAllBenchMarks(findCtx, patchRawData, db.Collection(stableRegionsColl)) - if err != nil { - return fmt.Errorf("Error getting energy statistics: %v", err) - } - - // Log energy stats output - prComment := generatePRComment(allEnergyStats, version) - log.Println("🧪 Performance Results") - log.Println(prComment) - - // Save for PR comment if it is a PR run - commitSHA := os.Getenv("HEAD_SHA") - if commitSHA != "" { - fmt.Printf("Version ID: %s\n", version) - fmt.Printf("Commit SHA: %s\n", commitSHA) // Use fmt to print to stdout - fmt.Println(prComment) - } - - return nil -} - -func findRawData(ctx context.Context, project string, version string, coll *mongo.Collection) ([]RawData, error) { - filter := bson.D{ - {"info.project", project}, - {"info.version", version}, - {"info.variant", "perf"}, - {"info.task_name", "perf"}, - } - - cursor, err := coll.Find(ctx, filter) - if err != nil { - log.Fatalf( - "Error retrieving raw data for version %q: %v", - version, - err, - ) - } - defer func() { - err = cursor.Close(ctx) - if err != nil { - log.Fatalf("Error closing cursor while retrieving raw data for version %q: %v", version, err) - } - }() - - log.Printf("Successfully retrieved %d docs from version %s.\n", cursor.RemainingBatchLength(), version) - - var rawData []RawData - err = cursor.All(ctx, &rawData) - if err != nil { - log.Fatalf( - "Error decoding raw data from version %q: %v", - version, - err, - ) - } - - return rawData, err -} - -// Find the most recent stable region of the mainline version for a specific test/measurement -func findLastStableRegion(ctx context.Context, project string, testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { - filter := bson.D{ - {"time_series_info.project", project}, - {"time_series_info.variant", "perf"}, - {"time_series_info.task", "perf"}, - {"time_series_info.test", testname}, - {"time_series_info.measurement", measurement}, - {"last", true}, - {"contexts", bson.D{{"$in", bson.A{"GoDriver perf task"}}}}, // TODO (GODRIVER-3102): Refactor perf context for project switching. - } - - findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}) - - var sr *StableRegion - err := coll.FindOne(ctx, filter, findOptions).Decode(&sr) - if err != nil { - return nil, err - } - return sr, nil -} - -// For a specific test and measurement -func getEnergyStatsForOneBenchmark(ctx context.Context, rd RawData, coll *mongo.Collection) ([]*EnergyStats, error) { - testname := rd.Info.TestName - var energyStats []*EnergyStats - - for i := range rd.Rollups.Stats { - project := rd.Info.Project - measName := rd.Rollups.Stats[i].Name - measVal := rd.Rollups.Stats[i].Val - - stableRegion, err := findLastStableRegion(ctx, project, testname, measName, coll) - if err != nil { - log.Fatalf( - "Error finding last stable region for test %q, measurement %q: %v", - testname, - measName, - err, - ) - } - - // The performance analyzer compares the measurement value from the patch to a stable region that succeeds the latest change point. - // For example, if there were 5 measurements since the last change point, then the stable region is the 5 latest values for the measurement. - stableRegionVec := mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values) - measValVec := mat.NewDense(1, 1, []float64{measVal}) // singleton - - estat, tstat, hscore, err := getEnergyStatistics(stableRegionVec, measValVec) - if err != nil { - log.Fatalf( - "Could not calculate energy stats for test %q, measurement %q: %v", - testname, - measName, - err, - ) - } - - zscore := getZScore(measVal, stableRegion.Mean, stableRegion.Std) - pChange := getPercentageChange(measVal, stableRegion.Mean) - - es := EnergyStats{ - Project: project, - Benchmark: testname, - Measurement: measName, - PatchVersion: rd.Info.Version, - StableRegion: *stableRegion, - MeasurementVal: measVal, - PercentChange: pChange, - EnergyStatistic: estat, - TestStatistic: tstat, - HScore: hscore, - ZScore: zscore, - } - energyStats = append(energyStats, &es) - } - - return energyStats, nil -} - -func getEnergyStatsForAllBenchMarks(ctx context.Context, patchRawData []RawData, coll *mongo.Collection) ([]*EnergyStats, error) { - var allEnergyStats []*EnergyStats - for _, rd := range patchRawData { - energyStats, err := getEnergyStatsForOneBenchmark(ctx, rd, coll) - if err != nil { - log.Fatalf( - "Could not get energy stats for %q: %v", - rd.Info.TestName, - err, - ) - } else { - allEnergyStats = append(allEnergyStats, energyStats...) - } - } - return allEnergyStats, nil -} - -func generatePRComment(energyStats []*EnergyStats, version string) string { - var comment strings.Builder - fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n\n", version) - - w := tabwriter.NewWriter(&comment, 0, 0, 1, ' ', 0) - fmt.Fprintln(w, "| Benchmark\t| Measurement\t| % Change\t| Patch Value\t| Stable Region\t| H-Score\t| Z-Score\t| ") - fmt.Fprintln(w, "| ---------\t| -----------\t| --------\t| -----------\t| -------------\t| -------\t| -------\t|") - - var significantEnergyStats []EnergyStats - for _, es := range energyStats { - // The "iterations" measurement is the number of iterations that the Go - // benchmark suite had to run to converge on a benchmark measurement. It - // is not comparable between benchmark runs, so is not a useful - // measurement to print here. Omit it. - if es.Measurement != "iterations" && math.Abs(es.ZScore) > 1.96 { - significantEnergyStats = append(significantEnergyStats, *es) - } - } - - if len(significantEnergyStats) == 0 { - comment.Reset() - fmt.Fprintf(&comment, "There were no significant changes to the performance to report for version %s.\n", version) - } else { - sort.Slice(significantEnergyStats, func(i, j int) bool { - return math.Abs(significantEnergyStats[i].PercentChange) > math.Abs(significantEnergyStats[j].PercentChange) - }) - for _, es := range significantEnergyStats { - fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t| %.4f\t|\n", es.Benchmark, es.Measurement, es.PercentChange, es.MeasurementVal, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.HScore, es.ZScore) - } - } - w.Flush() - - comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") - return comment.String() -} - -// Given two matrices, this function returns -// (e, t, h) = (E-statistic, test statistic, e-coefficient of inhomogeneity) -func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64, error) { - xrows, xcols := x.Dims() - yrows, ycols := y.Dims() - - if xcols != ycols { - return 0, 0, 0, fmt.Errorf("both inputs must have the same number of columns") - } - if xrows == 0 || yrows == 0 { - return 0, 0, 0, fmt.Errorf("inputs cannot be empty") - } - - xrowsf := float64(xrows) - yrowsf := float64(yrows) - - var A float64 // E|X-Y| - if xrowsf > 0 && yrowsf > 0 { - dist, err := getDistance(x, y) - if err != nil { - return 0, 0, 0, err - } - A = dist / (xrowsf * yrowsf) - } else { - A = 0 - } - - var B float64 // E|X-X'| - if xrowsf > 0 { - dist, err := getDistance(x, x) - if err != nil { - return 0, 0, 0, err - } - B = dist / (xrowsf * xrowsf) - } else { - B = 0 - } - - var C float64 // E|Y-Y'| - if yrowsf > 0 { - dist, err := getDistance(y, y) - if err != nil { - return 0, 0, 0, err - } - C = dist / (yrowsf * yrowsf) - } else { - C = 0 - } - - E := 2*A - B - C // D^2(F_x, F_y) - T := ((xrowsf * yrowsf) / (xrowsf + yrowsf)) * E - var H float64 - if A > 0 { - H = E / (2 * A) - } else { - H = 0 - } - return E, T, H, nil -} - -// Given two vectors (expected 1 col), -// this function returns the sum of distances between each pair. -func getDistance(x, y *mat.Dense) (float64, error) { - xrows, xcols := x.Dims() - yrows, ycols := y.Dims() - - if xcols != 1 || ycols != 1 { - return 0, fmt.Errorf("both inputs must be column vectors") - } - - var sum float64 - - for i := 0; i < xrows; i++ { - for j := 0; j < yrows; j++ { - sum += math.Abs(x.At(i, 0) - y.At(j, 0)) - } - } - return sum, nil -} - -// Get Z score for result x, compared to mean u and st dev o. -func getZScore(x, mu, sigma float64) float64 { - if sigma == 0 { - return math.NaN() - } - return (x - mu) / sigma -} - -// Get percentage change for result x compared to mean u. -func getPercentageChange(x, mu float64) float64 { - if mu == 0 { - return math.NaN() - } - return ((x - mu) / mu) * 100 -} diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go deleted file mode 100644 index a41e24687d..0000000000 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2025-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "gonum.org/v1/gonum/mat" -) - -func createTestVectors(start1 int, stop1 int, step1 int, start2 int, stop2 int, step2 int) (*mat.Dense, *mat.Dense) { - xData := []float64{} - yData := []float64{} - - for i := start1; i < stop1; i += step1 { - xData = append(xData, float64(i)) - } - for j := start2; j < stop2; j += step2 { - yData = append(yData, float64(j)) - } - - x := mat.NewDense(len(xData), 1, xData) - y := mat.NewDense(len(yData), 1, yData) - - return x, y -} - -func TestEnergyStatistics(t *testing.T) { - t.Run("similar distributions should have small e,t,h values ", func(t *testing.T) { - x, y := createTestVectors(1, 100, 1, 1, 105, 1) - e, tstat, h, _ := getEnergyStatistics(x, y) - - del := 1e-3 - // Limit precision of comparison to 3 digits after the decimal. - assert.InDelta(t, 0.160, e, del) // |0.160 - e| < 0.001 - assert.InDelta(t, 8.136, tstat, del) - assert.InDelta(t, 0.002, h, del) - }) - - t.Run("different distributions should have large e,t,h values", func(t *testing.T) { - x, y := createTestVectors(1, 100, 1, 10000, 13000, 14) - e, tstat, h, _ := getEnergyStatistics(x, y) - del := 1e-3 - - assert.InDelta(t, 21859.691, e, del) - assert.InDelta(t, 1481794.709, tstat, del) - assert.InDelta(t, 0.954, h, del) - }) - - t.Run("uni-variate distributions", func(t *testing.T) { - x, y := createTestVectors(1, 300, 1, 1000, 5000, 10) - e, tstat, h, _ := getEnergyStatistics(x, y) - del := 1e-3 - - assert.InDelta(t, 4257.009, e, del) - assert.InDelta(t, 728381.015, tstat, del) - assert.InDelta(t, 0.748, h, del) - }) - - t.Run("equal distributions should have all 0 values", func(t *testing.T) { - x := mat.NewDense(10, 1, []float64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) - y := mat.NewDense(1, 1, []float64{1}) - - e, tstat, h, _ := getEnergyStatistics(x, y) - - assert.Equal(t, 0.0, e) - assert.Equal(t, 0.0, tstat) - assert.Equal(t, 0.0, h) - }) - - t.Run("energy stats returns errors on malformed input", func(t *testing.T) { - x := mat.NewDense(2, 2, make([]float64, 4)) - y := mat.NewDense(2, 3, make([]float64, 6)) - - _, _, _, err := getEnergyStatistics(x, y) - assert.NotEqual(t, nil, err) - assert.ErrorContains(t, err, "both inputs must have the same number of columns") - - x.Reset() - y = &mat.Dense{} - - _, _, _, err = getEnergyStatistics(x, y) - assert.NotEqual(t, nil, err) - assert.ErrorContains(t, err, "inputs cannot be empty") - - x = mat.NewDense(2, 2, make([]float64, 4)) - y = mat.NewDense(3, 2, make([]float64, 6)) - - _, _, _, err = getEnergyStatistics(x, y) - assert.NotEqual(t, nil, err) - assert.ErrorContains(t, err, "both inputs must be column vectors") - }) -} diff --git a/internal/cmd/perfcomp/go.mod b/internal/cmd/perfcomp/go.mod deleted file mode 100644 index 2a3de5fd8b..0000000000 --- a/internal/cmd/perfcomp/go.mod +++ /dev/null @@ -1,34 +0,0 @@ -module go.mongodb.go/mongo-driver/v2/internal/cmd/perfcomp - -go 1.23.0 - -toolchain go1.23.10 - -replace go.mongodb.org/mongo-driver/v2 => ../../../ - -require ( - github.com/stretchr/testify v1.10.0 - go.mongodb.org/mongo-driver/v2 v2.2.2 - gonum.org/v1/gonum v0.16.0 -) - -require ( - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/spf13/pflag v1.0.6 // indirect -) - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/snappy v1.0.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/cobra v1.9.1 - github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.2 // indirect - github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - golang.org/x/crypto v0.33.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/text v0.23.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/internal/cmd/perfcomp/go.sum b/internal/cmd/perfcomp/go.sum deleted file mode 100644 index a226fa646b..0000000000 --- a/internal/cmd/perfcomp/go.sum +++ /dev/null @@ -1,64 +0,0 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= -github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go deleted file mode 100644 index c6d921f390..0000000000 --- a/internal/cmd/perfcomp/main.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2025-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package main - -import ( - "log" - - "github.com/spf13/cobra" -) - -func main() { - cmd := &cobra.Command{ - Use: "perfcomp", - Short: "perfcomp is a cli that reports stat-sig results between evergreen patches with the mainline commit", - Version: "0.0.0-alpha", - } - - cmd.AddCommand(newCompareCommand()) - cmd.AddCommand(newMdCommand()) - - if err := cmd.Execute(); err != nil { - log.Fatalf("error: %v", err) - } -} diff --git a/internal/cmd/perfcomp/mdreport.go b/internal/cmd/perfcomp/mdreport.go deleted file mode 100644 index b9611397d1..0000000000 --- a/internal/cmd/perfcomp/mdreport.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2025-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package main - -import ( - "bufio" - "fmt" - "log" - "net/url" - "os" - "strings" - - "github.com/spf13/cobra" -) - -const perfCompDir = "./internal/cmd/perfcomp/" -const perfReportFileTxt = "perf-report.txt" -const perfReportFileMd = "perf-report.md" -const perfVariant = "^perf$" -const hscoreDefLink = "https://en.wikipedia.org/wiki/Energy_distance#:~:text=E%2Dcoefficient%20of%20inhomogeneity" -const zscoreDefLink = "https://en.wikipedia.org/wiki/Standard_score#Calculation" - -func newMdCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "mdreport", - Short: "generates markdown output after run", - } - - cmd.Run = func(cmd *cobra.Command, args []string) { - if err := runMdCommand(cmd, args); err != nil { - log.Fatalf("failed to generate md: %v", err) - } - } - - return cmd -} - -func runMdCommand(cmd *cobra.Command, args []string) error { - var line string - - // open file to read - fRead, err := os.Open(perfCompDir + perfReportFileTxt) - if err != nil { - log.Fatalf("Could not open %s: %v", perfReportFileTxt, err) - } - defer fRead.Close() - - // open file to write - fWrite, err := os.Create(perfReportFileMd) - if err != nil { - log.Fatalf("Could not create %s: %v", perfReportFileMd, err) - } - defer fWrite.Close() - - fmt.Fprintf(fWrite, "## 🧪 Performance Results\n") - - // read the file line by line using scanner - scanner := bufio.NewScanner(fRead) - - var version string - var evgLink string - - for scanner.Scan() { - line = scanner.Text() - if strings.Contains(line, "Version ID:") { - // parse version - version = strings.Split(line, " ")[2] - } else if strings.Contains(line, "Commit SHA:") { - // parse commit SHA and write header - fmt.Fprintf(fWrite, "\n
\n%s\n\t
\n\n", line) - } else if strings.Contains(line, "version "+version) { - // dynamic Evergreen perf task link - evgLink, err = generateEvgLink(version, perfVariant) - if err != nil { - log.Println(err) - fmt.Fprintf(fWrite, "%s\n", line) - } else { - printUrlToLine(fWrite, line, evgLink, "version", -1) - } - } else if strings.Contains(line, "For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.") { - // last line of comment - evgLink, err = generateEvgLink(version, "") - if err != nil { - log.Println(err) - fmt.Fprintf(fWrite, "%s\n", line) - } else { - printUrlToLine(fWrite, line, evgLink, "Evergreen", 0) - } - } else if strings.Contains(line, ", ") { - line = strings.ReplaceAll(line, ", ", "
") - fmt.Fprintf(fWrite, "%s\n", line) - } else if strings.Contains(line, "H-Score") { - linkedWord := "[H-Score](" + hscoreDefLink + ")" - line = strings.ReplaceAll(line, "H-Score", linkedWord) - linkedWord = "[Z-Score](" + zscoreDefLink + ")" - line = strings.ReplaceAll(line, "Z-Score", linkedWord) - fmt.Fprintf(fWrite, "%s\n", line) - } else { - // all other regular lines - fmt.Fprintf(fWrite, "%s\n", line) - } - } - - fmt.Fprintf(fWrite, "
\n") - return nil -} - -func generateEvgLink(version string, variant string) (string, error) { - baseUrl := "https://spruce.mongodb.com" - page := "0" - sorts := "STATUS:ASC;BASE_STATUS:DESC" - - u, err := url.Parse(baseUrl) - if err != nil { - return "", fmt.Errorf("Error parsing URL: %v", err) - } - - u.Path = fmt.Sprintf("version/%s/tasks", version) - - // construct query parameters - queryParams := url.Values{} - queryParams.Add("page", page) - queryParams.Add("sorts", sorts) - if variant != "" { - queryParams.Add("variant", variant) - } - - u.RawQuery = queryParams.Encode() - return u.String(), nil -} - -func printUrlToLine(fWrite *os.File, line string, link string, targetWord string, step int) { - words := strings.Split(line, " ") - for i, w := range words { - if i > 0 && words[i+step] == targetWord { - fmt.Fprintf(fWrite, "[%s](%s)", w, link) - } else { - fmt.Fprint(fWrite, w) - } - - if i < len(words)-1 { - fmt.Fprint(fWrite, " ") - } else { - fmt.Fprint(fWrite, "\n") - } - } -}