fix existing tests

Author: Tofel

wasp/benchspy/TO_DO.md

@@ -1,6 +1,8 @@
 Known things to do:
 - [ ] add logger
-- [ ] add tests
+- [ ] add unit tests for prometheus
+- [ ] add wasp test for prometheus only
+- [ ] add e2e OCRv2 test with CTFv2
 - [ ] write documentation
 - [ ] add report builder (?)
 - [ ] add wrapper function for executing some code and then creating a report

wasp/benchspy/basic.go

@@ -50,6 +50,11 @@ func NewBasicData(commitOrTag string, generators ...*wasp.Generator) (*BasicData
 		b.GeneratorConfigs[g.Cfg.GenName] = g.Cfg
 	}
 
+	timeErr := b.FillStartEndTimes()
+	if timeErr != nil {
+		return nil, timeErr
+	}
+
 	return b, nil
 }
 

wasp/benchspy/generator.go

@@ -214,12 +214,14 @@ func (g *GeneratorQueryExecutor) MarshalJSON() ([]byte, error) {
 }
 
 func (g *GeneratorQueryExecutor) UnmarshalJSON(data []byte) error {
-	// helper struct with QueryExecutors as json.RawMessage
+	// helper struct with QueryExecutors as json.RawMessage and QueryResults as map[string]interface{}
+	// and as actual types
 	type Alias GeneratorQueryExecutor
 	var raw struct {
 		Alias
-		GeneratorCfg wasp.Config       `json:"generator_config"`
-		Queries      []json.RawMessage `json:"queries"`
+		GeneratorCfg wasp.Config            `json:"generator_config"`
+		Queries      []json.RawMessage      `json:"queries"`
+		QueryResults map[string]interface{} `json:"query_results"`
 	}
 
 	// unmarshal into the helper struct to populate other fields automatically
@@ -239,8 +241,15 @@ func (g *GeneratorQueryExecutor) UnmarshalJSON(data []byte) error {
 		queries[queryName] = nil
 	}
 
+	// convert map[string]interface{} to map[string]actualType
+	convertedTypes, conversionErr := convertQueryResults(raw.QueryResults)
+	if conversionErr != nil {
+		return conversionErr
+	}
+
 	*g = GeneratorQueryExecutor(raw.Alias)
 	g.Queries = queries
+	g.QueryResults = convertedTypes
 	g.Generator = &wasp.Generator{
 		Cfg: &raw.GeneratorCfg,
 	}

wasp/benchspy/generator_test.go

@@ -235,7 +235,8 @@ func TestBenchSpy_GeneratorQueryExecutor_Execute(t *testing.T) {
 		// 4 responses with ~150ms latency (150ms sleep + some execution overhead)
 		// and 2-3 responses with ~200ms latency (200ms sleep + some execution overhead)
 		// expected median latency: (150ms, 151ms>
-		resultsAsStrings := ResultsAs("string", []QueryExecutor{executor}, StandardQueryExecutor_Generator, string(MedianLatency), string(Percentile95Latency), string(ErrorRate))
+		resultsAsStrings, err := ResultsAs("string", []QueryExecutor{executor}, StandardQueryExecutor_Generator, string(MedianLatency), string(Percentile95Latency), string(ErrorRate))
+		assert.NoError(t, err)
 		require.Equal(t, 3, len(resultsAsStrings))
 
 		medianLatencyFloat, err := strconv.ParseFloat(resultsAsStrings[string(MedianLatency)], 64)
@@ -361,7 +362,7 @@ func TestBenchSpy_GeneratorQueryExecutor_MarshalJSON(t *testing.T) {
 		err = json.Unmarshal(data, &recovered)
 		assert.NoError(t, err)
 
-		assert.Equal(t, original.Kind, recovered.Kind)
+		assert.Equal(t, original.KindName, recovered.KindName)
 		assert.Equal(t, original.QueryResults, recovered.QueryResults)
 		assert.Equal(t, len(original.Queries), len(recovered.Queries))
 	})

wasp/benchspy/loki.go

@@ -2,6 +2,7 @@ package benchspy
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/url"
 	"reflect"
@@ -163,6 +164,30 @@ func (l *LokiQueryExecutor) TimeRange(start, end time.Time) {
 	l.EndTime = end
 }
 
+func (l *LokiQueryExecutor) UnmarshalJSON(data []byte) error {
+	// helper struct with QueryResults map[string]interface{}
+	type Alias LokiQueryExecutor
+	var raw struct {
+		Alias
+		QueryResults map[string]interface{} `json:"query_results"`
+	}
+
+	// unmarshal into the helper struct to populate other fields automatically
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return err
+	}
+
+	// convert map[string]interface{} to map[string]actualType
+	convertedTypes, conversionErr := convertQueryResults(raw.QueryResults)
+	if conversionErr != nil {
+		return conversionErr
+	}
+
+	*l = LokiQueryExecutor(raw.Alias)
+	l.QueryResults = convertedTypes
+	return nil
+}
+
 func NewStandardMetricsLokiExecutor(lokiConfig *wasp.LokiConfig, testName, generatorName, branch, commit string, startTime, endTime time.Time) (*LokiQueryExecutor, error) {
 	lq := &LokiQueryExecutor{
 		KindName:     string(StandardQueryExecutor_Loki),

wasp/benchspy/prometheus.go

@@ -2,6 +2,7 @@ package benchspy
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"time"
@@ -18,7 +19,7 @@ type PrometheusQueryExecutor struct {
 	startTime, endTime time.Time              `json:"-"`
 	client             *client.Prometheus     `json:"-"`
 	Queries            map[string]string      `json:"queries"`
-	ResourceResults    map[string]interface{} `json:"resources"`
+	QueryResults       map[string]interface{} `json:"query_results"`
 	warnings           map[string]v1.Warnings `json:"-"`
 }
 
@@ -30,12 +31,12 @@ func NewPrometheusQueryExecutor(url string, startTime, endTime time.Time, querie
 	}
 
 	return &PrometheusQueryExecutor{
-		KindName:        string(StandardQueryExecutor_Prometheus),
-		client:          c,
-		Queries:         queries,
-		startTime:       startTime,
-		endTime:         endTime,
-		ResourceResults: make(map[string]interface{}),
+		KindName:     string(StandardQueryExecutor_Prometheus),
+		client:       c,
+		Queries:      queries,
+		startTime:    startTime,
+		endTime:      endTime,
+		QueryResults: make(map[string]interface{}),
 	}, nil
 }
 
@@ -51,11 +52,11 @@ func NewStandardPrometheusQueryExecutor(url string, startTime, endTime time.Time
 	}
 
 	return &PrometheusQueryExecutor{
-		client:          c,
-		Queries:         standardQueries,
-		startTime:       startTime,
-		endTime:         endTime,
-		ResourceResults: make(map[string]interface{}),
+		client:       c,
+		Queries:      standardQueries,
+		startTime:    startTime,
+		endTime:      endTime,
+		QueryResults: make(map[string]interface{}),
 	}, nil
 }
 
@@ -70,14 +71,14 @@ func (r *PrometheusQueryExecutor) Execute(ctx context.Context) error {
 			r.warnings[name] = warnings
 		}
 
-		r.ResourceResults[name] = result
+		r.QueryResults[name] = result
 	}
 
 	return nil
 }
 
 func (r *PrometheusQueryExecutor) Results() map[string]interface{} {
-	return r.ResourceResults
+	return r.QueryResults
 }
 
 func (l *PrometheusQueryExecutor) Kind() string {
@@ -138,12 +139,12 @@ func (r *PrometheusQueryExecutor) Warnings() map[string]v1.Warnings {
 	return r.warnings
 }
 
-func (r *PrometheusQueryExecutor) MustResourcesAsValue() map[string]model.Value {
-	resources := make(map[string]model.Value)
-	for name, resource := range r.ResourceResults {
-		resources[name] = resource.(model.Value)
+func (r *PrometheusQueryExecutor) MustResultsAsValue() map[string]model.Value {
+	results := make(map[string]model.Value)
+	for name, result := range r.QueryResults {
+		results[name] = result.(model.Value)
 	}
-	return resources
+	return results
 }
 
 func (r *PrometheusQueryExecutor) TimeRange(startTime, endTime time.Time) {
@@ -184,3 +185,27 @@ func (r *PrometheusQueryExecutor) generateStandardQueries(nameRegexPattern strin
 
 	return standardQueries, nil
 }
+
+func (r *PrometheusQueryExecutor) UnmarshalJSON(data []byte) error {
+	// helper struct with QueryResults map[string]interface{}
+	type Alias PrometheusQueryExecutor
+	var raw struct {
+		Alias
+		QueryResults map[string]interface{} `json:"query_results"`
+	}
+
+	// unmarshal into the helper struct to populate other fields automatically
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return err
+	}
+
+	// convert map[string]interface{} to map[string]actualType
+	convertedTypes, conversionErr := convertQueryResults(raw.QueryResults)
+	if conversionErr != nil {
+		return conversionErr
+	}
+
+	*r = PrometheusQueryExecutor(raw.Alias)
+	r.QueryResults = convertedTypes
+	return nil
+}