add standard set of metrics/queries and a new test for them

Author: Tofel

wasp/benchspy/loki.go

@@ -159,3 +159,70 @@ func (l *LokiQueryExecutor) TimeRange(start, end time.Time) {
 	l.StartTime = start
 	l.EndTime = end
 }
+
+type StandardMetric string
+
+const (
+	AverageLatency      StandardMetric = "average_latency"
+	Percentile95Latency StandardMetric = "95th_percentile_latency"
+	ErrorRate           StandardMetric = "error_rate"
+)
+
+var standardMetrics = []StandardMetric{AverageLatency, Percentile95Latency, ErrorRate}
+
+func NewStandardMetricsLokiExecutor(lokiConfig *wasp.LokiConfig, testName, generatorName, branch, commit string, startTime, endTime time.Time) (*LokiQueryExecutor, error) {
+	standardQueries, queryErr := generateStandardLokiQueries(testName, generatorName, branch, commit, startTime, endTime)
+	if queryErr != nil {
+		return nil, queryErr
+	}
+
+	return &LokiQueryExecutor{
+		Kind:         "loki",
+		Queries:      standardQueries,
+		LokiConfig:   lokiConfig,
+		QueryResults: make(map[string][]string),
+	}, nil
+}
+
+func standardQuery(standardMetric StandardMetric, testName, generatorName, branch, commit string, startTime, endTime time.Time) (string, error) {
+	switch standardMetric {
+	case AverageLatency:
+		return fmt.Sprintf("avg_over_time({branch=~\"%s\", commit=~\"%s\", go_test_name=~\"%s\", test_data_type=~\"responses\", gen_name=~\"%s\"} | json| unwrap duration [10s]) by (go_test_name, gen_name) / 1e6", branch, commit, testName, generatorName), nil
+	case Percentile95Latency:
+		return fmt.Sprintf("quantile_over_time(0.95, {branch=~\"%s\", commit=~\"%s\", go_test_name=~\"%s\", test_data_type=~\"responses\", gen_name=~\"%s\"} | json| unwrap duration [10s]) by (go_test_name, gen_name) / 1e6", branch, commit, testName, generatorName), nil
+	case ErrorRate:
+		queryRange := calculateTimeRange(startTime, endTime)
+		return fmt.Sprintf("sum(max_over_time({branch=~\"%s\", commit=~\"%s\", go_test_name=~\"%s\", test_data_type=~\"stats\", gen_name=~\"%s\"} | json| unwrap failed [%s]) by (node_id, go_test_name, gen_name)) by (__stream_shard__)", branch, commit, testName, generatorName, queryRange), nil
+	default:
+		return "", fmt.Errorf("unsupported standard metric %s", standardMetric)
+	}
+}
+
+func generateStandardLokiQueries(testName, generatorName, branch, commit string, startTime, endTime time.Time) (map[string]string, error) {
+	standardQueries := make(map[string]string)
+
+	for _, metric := range standardMetrics {
+		query, err := standardQuery(metric, testName, generatorName, branch, commit, startTime, endTime)
+		if err != nil {
+			return nil, err
+		}
+		standardQueries[string(metric)] = query
+	}
+
+	return standardQueries, nil
+}
+
+func calculateTimeRange(startTime, endTime time.Time) string {
+	totalSeconds := int(endTime.Sub(startTime).Seconds())
+
+	var rangeStr string
+	if totalSeconds%3600 == 0 { // Exact hours
+		rangeStr = fmt.Sprintf("%dh", totalSeconds/3600)
+	} else if totalSeconds%60 == 0 { // Exact minutes
+		rangeStr = fmt.Sprintf("%dm", totalSeconds/60)
+	} else { // Use seconds for uneven intervals
+		rangeStr = fmt.Sprintf("%ds", totalSeconds)
+	}
+
+	return rangeStr
+}

wasp/benchspy/report.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/pkg/errors"
+	"github.com/smartcontractkit/chainlink-testing-framework/wasp"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -20,14 +22,20 @@ func (b *StandardReport) Store() (string, error) {
 	return b.LocalStorage.Store(b.TestName, b.CommitOrTag, b)
 }
 
-func (b *StandardReport) Load() error {
-	return b.LocalStorage.Load(b.TestName, b.CommitOrTag, b)
+func (b *StandardReport) Load(testName, commitOrTag string) error {
+	return b.LocalStorage.Load(testName, commitOrTag, b)
+}
+
+func (b *StandardReport) LoadLatest(testName string) error {
+	return b.LocalStorage.Load(testName, "", b)
 }
 
 func (b *StandardReport) FetchData(ctx context.Context) error {
-	startEndErr := b.BasicData.FillStartEndTimes()
-	if startEndErr != nil {
-		return startEndErr
+	if b.TestStart.IsZero() || b.TestEnd.IsZero() {
+		startEndErr := b.BasicData.FillStartEndTimes()
+		if startEndErr != nil {
+			return startEndErr
+		}
 	}
 
 	basicErr := b.BasicData.Validate()
@@ -91,6 +99,42 @@ func (b *StandardReport) IsComparable(otherReport Reporter) error {
 	return nil
 }
 
+func NewStandardReport(commitOrTag string, executionEnvironment ExecutionEnvironment, generators ...*wasp.Generator) (*StandardReport, error) {
+	basicData, basicErr := NewBasicData(commitOrTag, generators...)
+	if basicErr != nil {
+		return nil, errors.Wrapf(basicErr, "failed to create basic data for generators %v", generators)
+	}
+
+	startEndErr := basicData.FillStartEndTimes()
+	if startEndErr != nil {
+		return nil, startEndErr
+	}
+
+	var queryExecutors []QueryExecutor
+	for _, g := range generators {
+		if !generatorHasLabels(g) {
+			return nil, fmt.Errorf("generator %s is missing branch or commit labels", g.Cfg.GenName)
+		}
+		executor, executorErr := NewStandardMetricsLokiExecutor(g.Cfg.LokiConfig, basicData.TestName, g.Cfg.GenName, g.Cfg.Labels["branch"], g.Cfg.Labels["commit"], basicData.TestStart, basicData.TestEnd)
+		if executorErr != nil {
+			return nil, errors.Wrapf(executorErr, "failed to create standard Loki query executor for generator %s", g.Cfg.GenName)
+		}
+		queryExecutors = append(queryExecutors, executor)
+	}
+
+	return &StandardReport{
+		BasicData:      *basicData,
+		QueryExecutors: queryExecutors,
+		ResourceReporter: ResourceReporter{
+			ExecutionEnvironment: executionEnvironment,
+		},
+	}, nil
+}
+
+func generatorHasLabels(g *wasp.Generator) bool {
+	return g.Cfg.Labels["branch"] != "" && g.Cfg.Labels["commit"] != ""
+}
+
 func (s *StandardReport) UnmarshalJSON(data []byte) error {
 	// helper struct with QueryExecutors as json.RawMessage
 	type Alias StandardReport

wasp/benchspy/types.go

@@ -9,7 +9,9 @@ type Storer interface {
 	// Store stores the report in a persistent storage and returns the path to it, or an error
 	Store() (string, error)
 	// Load loads the report from a persistent storage and returns it, or an error
-	Load() error
+	Load(testName, commitOrTag string) error
+	// LoadLatest loads the latest report from a persistent storage and returns it, or an error
+	LoadLatest(testName string) error
 }
 
 type DataFetcher interface {

wasp/benchspy_test.go

@@ -61,7 +61,6 @@ func TestBenchSpyWithLokiQuery(t *testing.T) {
 	currentReport.QueryExecutors = append(currentReport.QueryExecutors, lokiQueryExecutor)
 
 	gen.Run(true)
-	currentReport.TestEnd = time.Now()
 
 	fetchCtx, cancelFn := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancelFn()
@@ -72,16 +71,14 @@ func TestBenchSpyWithLokiQuery(t *testing.T) {
 	// path, storeErr := currentReport.Store()
 	// require.NoError(t, storeErr, "failed to store current report", path)
 
+	// this is only needed, because we are using a non-standard directory
+	// otherwise, the Load method would be able to find the file
 	previousReport := benchspy.StandardReport{
-		BasicData: benchspy.BasicData{
-			TestName:    t.Name(),
-			CommitOrTag: "e7fc5826a572c09f8b93df3b9f674113372ce924",
-		},
 		LocalStorage: benchspy.LocalStorage{
 			Directory: "test_performance_reports",
 		},
 	}
-	loadErr := previousReport.Load()
+	loadErr := previousReport.Load(t.Name(), "e7fc5826a572c09f8b93df3b9f674113372ce924")
 	require.NoError(t, loadErr, "failed to load previous report")
 
 	isComparableErrs := previousReport.IsComparable(&currentReport)
@@ -110,7 +107,7 @@ func TestBenchSpyWithLokiQuery(t *testing.T) {
 		require.NoError(t, err, "failed to parse float")
 		previousSum += asFloat
 	}
-	previousAverage := currentSum / float64(len(previousReport.QueryExecutors[0].Results()["vu_over_time"]))
+	previousAverage := previousSum / float64(len(previousReport.QueryExecutors[0].Results()["vu_over_time"]))
 
 	require.Equal(t, currentAverage, previousAverage, "vu_over_time averages are not the same")
 }
@@ -167,16 +164,14 @@ func TestBenchSpyWithTwoLokiQueries(t *testing.T) {
 	// path, storeErr := currentReport.Store()
 	// require.NoError(t, storeErr, "failed to store current report", path)
 
+	// this is only needed, because we are using a non-standard directory
+	// otherwise, the Load method would be able to find the file
 	previousReport := benchspy.StandardReport{
-		BasicData: benchspy.BasicData{
-			TestName:    t.Name(),
-			CommitOrTag: "e7fc5826a572c09f8b93df3b9f674113372ce924",
-		},
 		LocalStorage: benchspy.LocalStorage{
 			Directory: "test_performance_reports",
 		},
 	}
-	loadErr := previousReport.Load()
+	loadErr := previousReport.Load(t.Name(), "e7fc5826a572c09f8b93df3b9f674113372ce924")
 	require.NoError(t, loadErr, "failed to load previous report")
 
 	isComparableErrs := previousReport.IsComparable(&currentReport)
@@ -235,3 +230,90 @@ func TestBenchSpyWithTwoLokiQueries(t *testing.T) {
 	diffPrecentage := (currentRespAverage - previousRespAverage) / previousRespAverage * 100
 	require.LessOrEqual(t, math.Abs(diffPrecentage), 1.0, "responses_over_time averages are more than 1% different", fmt.Sprintf("%.4f", diffPrecentage))
 }
+
+func TestBenchSpyWithStandardLokiMetrics(t *testing.T) {
+	label := "benchspy-std"
+
+	gen, err := wasp.NewGenerator(&wasp.Config{
+		T:          t,
+		LokiConfig: wasp.NewEnvLokiConfig(),
+		GenName:    "vu",
+		Labels: map[string]string{
+			"branch": label,
+			"commit": label,
+		},
+		CallTimeout: 100 * time.Millisecond,
+		LoadType:    wasp.VU,
+		Schedule: wasp.CombineAndRepeat(
+			2,
+			wasp.Steps(10, 1, 10, 10*time.Second),
+			wasp.Plain(30, 15*time.Second),
+			wasp.Steps(20, -1, 10, 5*time.Second),
+		),
+		VU: wasp.NewMockVU(&wasp.MockVirtualUserConfig{
+			CallSleep: 50 * time.Millisecond,
+		}),
+	})
+	require.NoError(t, err)
+
+	gen.Run(true)
+
+	currentReport, err := benchspy.NewStandardReport("e7fc5826a572c09f8b93df3b9f674113372ce925", benchspy.ExecutionEnvironment_Docker, gen)
+	require.NoError(t, err)
+
+	fetchCtx, cancelFn := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancelFn()
+
+	fetchErr := currentReport.FetchData(fetchCtx)
+	require.NoError(t, fetchErr, "failed to fetch current report")
+
+	// path, storeErr := currentReport.Store()
+	// require.NoError(t, storeErr, "failed to store current report", path)
+
+	// this is only needed, because we are using a non-standard directory
+	// otherwise, the Load method would be able to find the file
+	previousReport := benchspy.StandardReport{
+		LocalStorage: benchspy.LocalStorage{
+			Directory: "test_performance_reports",
+		},
+	}
+	loadErr := previousReport.Load(t.Name(), "e7fc5826a572c09f8b93df3b9f674113372ce924")
+	require.NoError(t, loadErr, "failed to load previous report")
+
+	isComparableErrs := previousReport.IsComparable(currentReport)
+	require.Empty(t, isComparableErrs, "reports were not comparable", isComparableErrs)
+
+	var compareAverages = func(metricName benchspy.StandardMetric) {
+		require.NotEmpty(t, currentReport.QueryExecutors[0].Results()[string(metricName)], "%s results were missing from current report", string(metricName))
+		require.NotEmpty(t, previousReport.QueryExecutors[0].Results()[string(metricName)], "%s results were missing from previous report", string(metricName))
+		require.Equal(t, len(currentReport.QueryExecutors[0].Results()[string(metricName)]), len(previousReport.QueryExecutors[0].Results()[string(metricName)]), "%s results are not the same length", string(metricName))
+
+		var currentAvgSum float64
+		for _, value := range currentReport.QueryExecutors[0].Results()[string(metricName)] {
+			asFloat, err := strconv.ParseFloat(value, 64)
+			require.NoError(t, err, "failed to parse float")
+			currentAvgSum += asFloat
+		}
+		currentAvgAverage := currentAvgSum / float64(len(currentReport.QueryExecutors[0].Results()[string(metricName)]))
+
+		var previousAvgSum float64
+		for _, value := range previousReport.QueryExecutors[0].Results()[string(metricName)] {
+			asFloat, err := strconv.ParseFloat(value, 64)
+			require.NoError(t, err, "failed to parse float")
+			previousAvgSum += asFloat
+		}
+		previousAvgAverage := previousAvgSum / float64(len(previousReport.QueryExecutors[0].Results()[string(metricName)]))
+
+		var diffPrecentage float64
+		if previousAvgAverage != 0 {
+			diffPrecentage = (currentAvgAverage - previousAvgAverage) / previousAvgAverage * 100
+		} else {
+			diffPrecentage = currentAvgAverage * 100
+		}
+		require.LessOrEqual(t, math.Abs(diffPrecentage), 1.0, "%s averages are more than 1% different", string(metricName), fmt.Sprintf("%.4f", diffPrecentage))
+	}
+
+	compareAverages(benchspy.AverageLatency)
+	compareAverages(benchspy.Percentile95Latency)
+	compareAverages(benchspy.ErrorRate)
+}

wasp/go.mod

@@ -2,8 +2,6 @@ module github.com/smartcontractkit/chainlink-testing-framework/wasp
 
 go 1.23
 
-toolchain go1.23.3
-
 require (
 	github.com/K-Phoen/grabana v0.22.2
 	github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8