add plain-segment-only notes/comments

Author: Tofel

wasp/benchspy/TO_DO.md

@@ -6,6 +6,4 @@ Known things to do:
 - [ ] test with k8s app (focus: resources)
 - [ ] add report builder (?)
 - [ ] add wrapper function for executing some code and then creating a report
-- [ ] think what to do with errors... do we really need a slice?
-- [ ] if TestEnd is zero, default to `time.Now()`?
 - [ ] add helper method for a profile what would create a report based on all generators?

wasp/benchspy/loki.go

@@ -35,8 +35,6 @@ type LokiQueryExecutor struct {
 	// Performance queries results
 	// can be anything, avg RPS, amount of errors, 95th percentile of CPU utilization, etc
 	QueryResults map[string][]string `json:"query_results"`
-	// In case something went wrong, but sure we need it
-	Errors []error `json:"errors"`
 
 	LokiConfig *wasp.LokiConfig `json:"-"`
 }
@@ -192,6 +190,8 @@ func standardQuery(standardMetric StandardMetric, testName, generatorName, branc
 		return fmt.Sprintf("quantile_over_time(0.95, {branch=~\"%s\", commit=~\"%s\", go_test_name=~\"%s\", test_data_type=~\"responses\", gen_name=~\"%s\"} | json| unwrap duration [10s]) by (go_test_name, gen_name) / 1e6", branch, commit, testName, generatorName), nil
 	case ErrorRate:
 		queryRange := calculateTimeRange(startTime, endTime)
+		// this becomes problematic if we want to only consider plain segments, because each might have a different length and thus should have a different range window for accurate calculation
+		// unless... we will are only interested in comparing the differences between reports, not the actual values, then it won't matter that error rate is skewed (calculated over ranges longer than query interval)
 		return fmt.Sprintf("sum(max_over_time({branch=~\"%s\", commit=~\"%s\", go_test_name=~\"%s\", test_data_type=~\"stats\", gen_name=~\"%s\"} | json| unwrap failed [%s]) by (node_id, go_test_name, gen_name)) by (__stream_shard__)", branch, commit, testName, generatorName, queryRange), nil
 	default:
 		return "", fmt.Errorf("unsupported standard metric %s", standardMetric)

wasp/benchspy/report.go

@@ -47,6 +47,11 @@ func (b *StandardReport) FetchData(ctx context.Context) error {
 
 	for _, queryExecutor := range b.QueryExecutors {
 		errGroup.Go(func() error {
+			// feature: PLAIN SEGEMENT ONLY
+			// go over all schedules and execute the code below only for ones with type "plain"
+			// and then concatenate that data and return that; if parallelizing then we should first
+			// create a slice of plain segments and then, when sending results over channel include the index,
+			// so that we can concatenate them in the right order
 			queryExecutor.TimeRange(b.TestStart, b.TestEnd)
 
 			if validateErr := queryExecutor.Validate(); validateErr != nil {