remove ResourceFetcher, now Prometheus is just another QueryExecutor

Author: Tofel

wasp/benchspy/TO_DO.md

@@ -2,8 +2,6 @@ Known things to do:
 - [ ] add logger
 - [ ] add tests
 - [ ] write documentation
-- [ ] test with Docker app (focus: resources)
-- [ ] test with k8s app (focus: resources)
 - [ ] add report builder (?)
 - [ ] add wrapper function for executing some code and then creating a report
 - [ ] add helper method for a profile what would create a report based on all generators?

wasp/benchspy/generator.go

@@ -14,15 +14,15 @@ import (
 type GeneratorQueryFn = func(responses *wasp.SliceBuffer[wasp.Response]) (string, error)
 
 type GeneratorQueryExecutor struct {
-	Kind         string                      `json:"kind"`
+	KindName     string                      `json:"kind"`
 	Generator    *wasp.Generator             `json:"generator_config"`
 	Queries      map[string]GeneratorQueryFn `json:"queries"`
-	QueryResults map[string][]string         `json:"query_results"`
+	QueryResults map[string]interface{}      `json:"query_results"`
 }
 
 func NewGeneratorQueryExecutor(generator *wasp.Generator) (*GeneratorQueryExecutor, error) {
 	g := &GeneratorQueryExecutor{
-		Kind:      "generator",
+		KindName:  string(StandardQueryExecutor_Generator),
 		Generator: generator,
 	}
 
@@ -32,15 +32,19 @@ func NewGeneratorQueryExecutor(generator *wasp.Generator) (*GeneratorQueryExecut
 	}
 
 	g.Queries = queries
-	g.QueryResults = make(map[string][]string)
+	g.QueryResults = make(map[string]interface{})
 
 	return g, nil
 }
 
-func (g *GeneratorQueryExecutor) Results() map[string][]string {
+func (g *GeneratorQueryExecutor) Results() map[string]interface{} {
 	return g.QueryResults
 }
 
+func (l *GeneratorQueryExecutor) Kind() string {
+	return l.KindName
+}
+
 func (g *GeneratorQueryExecutor) IsComparable(otherQueryExecutor QueryExecutor) error {
 	otherType := reflect.TypeOf(otherQueryExecutor)
 
@@ -113,7 +117,7 @@ func (g *GeneratorQueryExecutor) Execute(_ context.Context) error {
 			return queryErr
 		}
 
-		g.QueryResults[queryName] = []string{results}
+		g.QueryResults[queryName] = results
 	}
 
 	return nil
@@ -184,14 +188,14 @@ func (g *GeneratorQueryExecutor) standardQuery(standardMetric StandardLoadMetric
 func (g *GeneratorQueryExecutor) MarshalJSON() ([]byte, error) {
 	// we need custom marshalling to only include query names, since the functions are not serializable
 	type QueryExecutor struct {
-		Kind         string              `json:"kind"`
-		Generator    interface{}         `json:"generator_config"`
-		Queries      []string            `json:"queries"`
-		QueryResults map[string][]string `json:"query_results"`
+		Kind         string                 `json:"kind"`
+		Generator    interface{}            `json:"generator_config"`
+		Queries      []string               `json:"queries"`
+		QueryResults map[string]interface{} `json:"query_results"`
 	}
 
 	return json.Marshal(&QueryExecutor{
-		Kind: g.Kind,
+		Kind: g.KindName,
 		Generator: func() interface{} {
 			if g.Generator != nil {
 				return g.Generator.Cfg

wasp/benchspy/generator_test.go

@@ -30,7 +30,7 @@ func TestBenchSpy_NewGeneratorQueryExecutor(t *testing.T) {
 		}
 		executor, err := NewGeneratorQueryExecutor(gen)
 		assert.NoError(t, err)
-		assert.Equal(t, "generator", executor.Kind)
+		assert.Equal(t, "generator", executor.KindName)
 		assert.Equal(t, gen, executor.Generator)
 		assert.NotEmpty(t, executor.Queries)
 		assert.NotNil(t, executor.QueryResults)
@@ -49,8 +49,8 @@ func TestBenchSpy_NewGeneratorQueryExecutor(t *testing.T) {
 }
 
 func TestBenchSpy_GeneratorQueryExecutor_Results(t *testing.T) {
-	expected := map[string][]string{
-		"test": {"result"},
+	expected := map[string]interface{}{
+		"test": "result",
 	}
 	executor := &GeneratorQueryExecutor{
 		QueryResults: expected,
@@ -235,27 +235,24 @@ func TestBenchSpy_GeneratorQueryExecutor_Execute(t *testing.T) {
 		// 4 responses with ~150ms latency (150ms sleep + some execution overhead)
 		// and 2-3 responses with ~200ms latency (200ms sleep + some execution overhead)
 		// expected median latency: (150ms, 151ms>
-		medianLatency, exists := results[string(MedianLatency)]
-		assert.True(t, exists)
+		resultsAsStrings := ResultsAs("string", []QueryExecutor{executor}, StandardQueryExecutor_Generator, string(MedianLatency), string(Percentile95Latency), string(ErrorRate))
+		require.Equal(t, 3, len(resultsAsStrings))
 
-		medianLatencyFloat, err := strconv.ParseFloat(medianLatency[0], 64)
+		medianLatencyFloat, err := strconv.ParseFloat(resultsAsStrings[string(MedianLatency)], 64)
 		assert.NoError(t, err)
 		require.InDelta(t, 151.0, medianLatencyFloat, 1.0)
 
-		p95Latency, exists := results[string(Percentile95Latency)]
-		assert.True(t, exists)
-
 		// since we have 2-3 responses with 200-201ms latency, the 95th percentile should be (200ms, 201ms>
-		p95LatencyFloat, err := strconv.ParseFloat(p95Latency[0], 64)
+		p95LatencyFloat, err := strconv.ParseFloat(resultsAsStrings[string(Percentile95Latency)], 64)
 		assert.NoError(t, err)
 		require.InDelta(t, 201.0, p95LatencyFloat, 1.0)
 
-		errorRate, exists := results[string(ErrorRate)]
+		errorRate, exists := resultsAsStrings[string(ErrorRate)]
 		assert.True(t, exists)
 
 		// error rate is the number of failures divided by the total number of responses
 		expectedErrorRate := float64(actualFailures) / (float64(fakeGun.maxSuccesses) + float64(actualFailures))
-		assert.Equal(t, []string{fmt.Sprintf("%.4f", expectedErrorRate)}, errorRate)
+		assert.Equal(t, fmt.Sprintf("%.4f", expectedErrorRate), errorRate)
 	})
 
 	t.Run("all responses failed", func(t *testing.T) {
@@ -345,15 +342,15 @@ func TestBenchSpy_GeneratorQueryExecutor_MarshalJSON(t *testing.T) {
 			},
 		}
 		original, _ := NewGeneratorQueryExecutor(gen)
-		original.QueryResults["test"] = []string{"result"}
-		original.QueryResults["test2"] = []string{"1", "2", "3"}
+		original.QueryResults["test"] = "result"
+		original.QueryResults["test2"] = "1"
 
 		original.Queries = map[string]GeneratorQueryFn{
 			"test": func(responses *wasp.SliceBuffer[wasp.Response]) (string, error) {
 				return "result", nil
 			},
 			"test2": func(responses *wasp.SliceBuffer[wasp.Response]) (string, error) {
-				return "1,2,3", nil
+				return "1", nil
 			},
 		}
 

wasp/benchspy/loki.go

@@ -16,15 +16,15 @@ import (
 
 func NewLokiQueryExecutor(queries map[string]string, lokiConfig *wasp.LokiConfig) *LokiQueryExecutor {
 	return &LokiQueryExecutor{
-		Kind:         "loki",
+		KindName:     string(StandardQueryExecutor_Loki),
 		Queries:      queries,
 		Config:       lokiConfig,
-		QueryResults: make(map[string][]string),
+		QueryResults: make(map[string]interface{}),
 	}
 }
 
 type LokiQueryExecutor struct {
-	Kind string `json:"kind"`
+	KindName string `json:"kind"`
 	// Test metrics
 	StartTime time.Time `json:"start_time"`
 	EndTime   time.Time `json:"end_time"`
@@ -34,15 +34,19 @@ type LokiQueryExecutor struct {
 	Queries map[string]string `json:"queries"`
 	// Performance queries results
 	// can be anything, avg RPS, amount of errors, 95th percentile of CPU utilization, etc
-	QueryResults map[string][]string `json:"query_results"`
+	QueryResults map[string]interface{} `json:"query_results"`
 
 	Config *wasp.LokiConfig `json:"-"`
 }
 
-func (l *LokiQueryExecutor) Results() map[string][]string {
+func (l *LokiQueryExecutor) Results() map[string]interface{} {
 	return l.QueryResults
 }
 
+func (l *LokiQueryExecutor) Kind() string {
+	return l.KindName
+}
+
 func (l *LokiQueryExecutor) IsComparable(otherQueryExecutor QueryExecutor) error {
 	otherType := reflect.TypeOf(otherQueryExecutor)
 
@@ -65,16 +69,23 @@ func (l *LokiQueryExecutor) Validate() error {
 }
 
 func (l *LokiQueryExecutor) Execute(ctx context.Context) error {
-	splitAuth := strings.Split(l.Config.BasicAuth, ":")
 	var basicAuth client.LokiBasicAuth
-	if len(splitAuth) == 2 {
-		basicAuth = client.LokiBasicAuth{
-			Login:    splitAuth[0],
-			Password: splitAuth[1],
+
+	if l.Config == nil {
+		return errors.New("loki config is missing. Please set it and try again")
+	}
+
+	if l.Config.BasicAuth != "" {
+		splitAuth := strings.Split(l.Config.BasicAuth, ":")
+		if len(splitAuth) == 2 {
+			basicAuth = client.LokiBasicAuth{
+				Login:    splitAuth[0],
+				Password: splitAuth[1],
+			}
 		}
 	}
 
-	l.QueryResults = make(map[string][]string)
+	l.QueryResults = make(map[string]interface{})
 	resultCh := make(chan map[string][]string, len(l.Queries))
 	errGroup, errCtx := errgroup.WithContext(ctx)
 
@@ -154,9 +165,9 @@ func (l *LokiQueryExecutor) TimeRange(start, end time.Time) {
 
 func NewStandardMetricsLokiExecutor(lokiConfig *wasp.LokiConfig, testName, generatorName, branch, commit string, startTime, endTime time.Time) (*LokiQueryExecutor, error) {
 	lq := &LokiQueryExecutor{
-		Kind:         "loki",
+		KindName:     string(StandardQueryExecutor_Loki),
 		Config:       lokiConfig,
-		QueryResults: make(map[string][]string),
+		QueryResults: make(map[string]interface{}),
 	}
 
 	standardQueries, queryErr := lq.generateStandardQueries(testName, generatorName, branch, commit, startTime, endTime)

wasp/benchspy/loki_test.go

@@ -193,16 +193,16 @@ func TestBenchSpy_NewLokiQueryExecutor(t *testing.T) {
 	}
 
 	executor := NewLokiQueryExecutor(queries, config)
-	assert.Equal(t, "loki", executor.Kind)
+	assert.Equal(t, "loki", executor.KindName)
 	assert.Equal(t, queries, executor.Queries)
 	assert.Equal(t, config, executor.Config)
 	assert.NotNil(t, executor.QueryResults)
 }
 
 func TestBenchSpy_LokiQueryExecutor_Results(t *testing.T) {
 	executor := &LokiQueryExecutor{
-		QueryResults: map[string][]string{
-			"test": {"result1", "result2"},
+		QueryResults: map[string]interface{}{
+			"test": []string{"result1", "result2"},
 		},
 	}
 	results := executor.Results()
@@ -211,13 +211,17 @@ func TestBenchSpy_LokiQueryExecutor_Results(t *testing.T) {
 
 type anotherQueryExecutor struct{}
 
+func (a *anotherQueryExecutor) Kind() string {
+	return "another"
+}
+
 func (a *anotherQueryExecutor) Validate() error {
 	return nil
 }
 func (a *anotherQueryExecutor) Execute(_ context.Context) error {
 	return nil
 }
-func (a *anotherQueryExecutor) Results() map[string][]string {
+func (a *anotherQueryExecutor) Results() map[string]interface{} {
 	return nil
 }
 func (a *anotherQueryExecutor) IsComparable(_ QueryExecutor) error {
@@ -336,7 +340,11 @@ func TestBenchSpy_LokiQueryExecutor_Execute(t *testing.T) {
 	err := executor.Execute(context.Background())
 	assert.NoError(t, err)
 	assert.Contains(t, executor.QueryResults, "test_query")
-	assert.Equal(t, "Log message 1", executor.QueryResults["test_query"][0])
+
+	asStringSlice, ok := executor.QueryResults["test_query"].([]string)
+	assert.True(t, ok)
+
+	assert.Equal(t, "Log message 1", asStringSlice[0])
 }
 
 func TestBenchSpy_LokiQueryExecutor_TimeRange(t *testing.T) {
@@ -365,7 +373,7 @@ func TestBenchSpy_NewStandardMetricsLokiExecutor(t *testing.T) {
 	executor, err := NewStandardMetricsLokiExecutor(config, testName, genName, branch, commit, start, end)
 	assert.NoError(t, err)
 	assert.NotNil(t, executor)
-	assert.Equal(t, "loki", executor.Kind)
+	assert.Equal(t, "loki", executor.KindName)
 	assert.Len(t, executor.Queries, len(standardLoadMetrics))
 }