DRIVERS-3254: Add perfcomp to drivers-evergreen-tools (#677)

Author: zhouselena

.evergreen/perfcomp/README.md

@@ -0,0 +1,87 @@
+# perfcomp
+
+**perfcomp** is a performance analyzer on a PR commit basis.
+
+## 📦 Installation
+
+To install the latest version:
+
+```bash
+go install github.com/mongodb-labs/drivers-evergreen-tools/perfcomp/cmd/perfcomp@latest
+```
+
+Or build it locally in `bin/perfcomp`:
+
+```bash
+bash build.sh
+```
+
+## 🔧 Usage
+
+### Parameters
+
+To use `perfcomp`, you should have an analytics node URI env variable called `PERF_URI_PRIVATE_ENDPOINT`. You can request for it from the devprod performance team.
+
+To run in your project repository, you need to create a [performance context](https://performance-monitoring-and-analysis.server-tig.prod.corp.mongodb.com/contexts) that captures all benchmarks in your project. This needs to be a triage context. Feel free to refer to the [Go Driver context](https://performance-monitoring-and-analysis.server-tig.prod.corp.mongodb.com/context/name/GoDriver%20perf%20task) as a template.
+
+> _If you are creating a triage context for the first time, it may take a few hours for your project's data to be tagged._
+
+You also need the name of the performance task and variant specific to your project. You can do a query in the analytics node `raw_results` collection:
+
+```
+db.raw_results.find({
+  “info.project”: “<project>”,
+  “info.version”: “<random_evergreen_version>"
+})
+```
+
+and look for the `variant` and `task_name` properties.
+
+### perfcomp CLI
+
+```bash
+perfcomp is a cli that reports stat-sig results between evergreen patches with the mainline commit
+
+Usage:
+  perfcomp [command]
+
+Available Commands:
+  compare     compare evergreen patch to mainline commit
+  mdreport    generates markdown output after run
+```
+
+### Commands
+
+#### compare
+
+```bash
+compare evergreen patch to mainline commit
+
+Usage:
+  perfcomp compare [version_id] [flags]
+
+Flags:
+  --perf-context string   specify the performance triage context, ex. "GoDriver perf task" (required)
+  --project      string   specify the name of an existing Evergreen project, ex. "mongo-go-driver" (required)
+  --task         string   specify the evergreen perf task name, ex. "perf" (required)
+  --variant      string   specify the perf task variant, ex. "perf" (required)
+```
+
+#### mdreport
+
+```bash
+generates markdown output after compare run (must be run after `compare`)
+
+Usage:
+  perfcomp mdreport
+```
+
+### Run via shell script
+
+Alternatively, you can run the perfcomp shell script. This script will run build and then run `compare`. From the root directory,
+
+```bash
+PERF_URI_PRIVATE_ENDPOINT="<perf_uri>" VERSION_ID="<version>" PROJECT="<project>" CONTEXT="<context>" TASK="<task>" VARIANT="<variant>" .evergreen/run-perf-comp.sh
+```
+
+If you would like to see a markdown preview of the report, you can also pass in `HEAD_SHA=""`. This will generate `.evergreen/perfcomp/perf-report.md`.

.evergreen/perfcomp/build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BIN_DIR="bin"
+mkdir -p $BIN_DIR
+go build -o $BIN_DIR/perfcomp ./cmd/perfcomp/

.evergreen/perfcomp/cmd/perfcomp/compare.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"math"
+	"os"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/mongodb-labs/drivers-evergreen-tools/perfcomp"
+	"github.com/spf13/cobra"
+)
+
+func newCompareCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "compare",
+		Short: "compare evergreen patch to mainline commit",
+		// Version id is a required argument
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) < 1 {
+				return fmt.Errorf("this command requires an evergreen patch version ID")
+			}
+			return nil
+		},
+	}
+
+	var project, task, variant, perfcontext string
+	cmd.Flags().StringVar(&project, "project", "", `specify the name of an existing Evergreen project, ex. "mongo-go-driver"`)
+	cmd.Flags().StringVar(&perfcontext, "perf-context", "", `specify the performance triage context, ex. "GoDriver perf task"`)
+	// TODO(DRIVERS-3264): Use first task / variant of the project by default for perf filtering
+	cmd.Flags().StringVar(&task, "task", "", `specify the evergreen performance task name, ex. "perf"`)
+	cmd.Flags().StringVar(&variant, "variant", "", `specify the performance variant, ex. "perf"`)
+
+	for _, flag := range []string{"project", "task", "variant", "context"} {
+		cmd.MarkFlagRequired(flag)
+	}
+
+	cmd.Run = func(cmd *cobra.Command, args []string) {
+		// Check for variables
+		uri := os.Getenv("PERF_URI_PRIVATE_ENDPOINT")
+		if uri == "" {
+			log.Fatal("PERF_URI_PRIVATE_ENDPOINT env variable is not set")
+		}
+
+		// Validate all flags
+		for _, flag := range []string{"project", "task", "variant", "perf-context"} {
+			if flag == "" {
+				log.Fatalf("must provide %s", flag)
+			}
+		}
+
+		// Run compare function
+		err := runCompare(cmd, args,
+			perfcomp.WithProject(project),
+			perfcomp.WithTask(task),
+			perfcomp.WithVariant(variant),
+			perfcomp.WithContext(perfcontext),
+		)
+		if err != nil {
+			log.Fatalf("failed to compare: %v", err)
+		}
+	}
+
+	return cmd
+}
+
+func createComment(result perfcomp.CompareResult) string {
+	var comment strings.Builder
+
+	if len(result.SigEnergyStats) == 0 {
+		comment.Reset()
+		fmt.Fprintf(&comment, "There were no significant changes to the performance to report for version %s.\n", result.Version)
+	} else {
+		fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n\n", result.Version)
+
+		w := tabwriter.NewWriter(&comment, 0, 0, 1, ' ', 0)
+
+		fmt.Fprintln(w, "| Benchmark\t| Measurement\t| % Change\t| Patch Value\t| Stable Region\t| H-Score\t| Z-Score\t| ")
+		fmt.Fprintln(w, "| ---------\t| -----------\t| --------\t| -----------\t| -------------\t| -------\t| -------\t|")
+
+		sort.Slice(result.SigEnergyStats, func(i, j int) bool {
+			return math.Abs(result.SigEnergyStats[i].PercentChange) > math.Abs(result.SigEnergyStats[j].PercentChange)
+		})
+		for _, es := range result.SigEnergyStats {
+			fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t| %.4f\t|\n",
+				es.Benchmark,
+				es.Measurement,
+				es.PercentChange,
+				es.MeasurementVal,
+				es.StableRegion.Mean,
+				es.StableRegion.Median,
+				es.StableRegion.Std,
+				es.HScore,
+				es.ZScore,
+			)
+		}
+
+		w.Flush()
+	}
+
+	comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*")
+	return comment.String()
+
+}
+
+func runCompare(cmd *cobra.Command, args []string, opts ...perfcomp.CompareOption) error {
+	perfAnalyticsConnString := os.Getenv("PERF_URI_PRIVATE_ENDPOINT")
+	version := args[len(args)-1]
+	opts = append(opts, perfcomp.WithVersion(version))
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second)
+	defer cancel()
+
+	res, err := perfcomp.Compare(ctx, perfAnalyticsConnString, opts...)
+	if err != nil {
+		log.Fatalf("failed to compare: %v", err)
+	}
+
+	res.CommitSHA = os.Getenv("HEAD_SHA")
+	res.MainlineCommit = os.Getenv("BASE_SHA")
+
+	prComment := createComment(*res)
+	log.Println("🧪 Performance Results")
+	log.Println(prComment)
+
+	if res.CommitSHA != "" {
+		// Write results to .txt file to parse into markdown comment
+		fWrite, err := os.Create(perfReportFileTxt)
+		if err != nil {
+			log.Fatalf("Could not create %s: %v", perfReportFileTxt, err)
+		}
+		defer fWrite.Close()
+
+		fmt.Fprintf(fWrite, "Version ID: %s\n", version)
+		fmt.Fprintf(fWrite, "Commit SHA: %s\n", res.CommitSHA)
+		fmt.Fprintln(fWrite, prComment)
+		log.Printf("PR commit %s: saved to %s for markdown comment.\n", res.CommitSHA, perfReportFileTxt)
+	}
+
+	return nil
+}

.evergreen/perfcomp/cmd/perfcomp/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"log"
+
+	"github.com/spf13/cobra"
+)
+
+func main() {
+	cmd := &cobra.Command{
+		Use:     "perfcomp",
+		Short:   "perfcomp is a cli that reports stat-sig results between evergreen patches with the mainline commit",
+		Version: "0.0.0-alpha",
+	}
+
+	cmd.AddCommand(newCompareCommand())
+	cmd.AddCommand(newMdCommand())
+
+	if err := cmd.Execute(); err != nil {
+		log.Fatalf("error: %v", err)
+	}
+}