Skip to content

Commit 1d5ca0c

Browse files
TofelTofel
committed
make the tool more abstract
1 parent 605856c commit 1d5ca0c

File tree

10 files changed

+809
-337
lines changed

10 files changed

+809
-337
lines changed

‎wasp/benchspy/basic.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package benchspy
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"time"
7+
8+
"github.com/pkg/errors"
9+
"github.com/smartcontractkit/chainlink-testing-framework/wasp"
10+
)
11+
12+
// BasicData is the basic data that is required for a report, common to all reports
13+
type BasicData struct {
14+
TestName string `json:"test_name"`
15+
CommitOrTag string `json:"commit_or_tag"`
16+
17+
// Test metrics
18+
TestStart time.Time `json:"test_start_timestamp"`
19+
TestEnd time.Time `json:"test_end_timestamp"`
20+
21+
// all, generator settings, including segments
22+
GeneratorConfigs map[string]*wasp.Config `json:"generator_configs"`
23+
}
24+
25+
func (b *BasicData) Validate() error {
26+
if b.TestStart.IsZero() {
27+
return errors.New("test start time is missing. We cannot query Loki without a time range. Please set it and try again")
28+
}
29+
if b.TestEnd.IsZero() {
30+
return errors.New("test end time is missing. We cannot query Loki without a time range. Please set it and try again")
31+
}
32+
33+
if len(b.GeneratorConfigs) == 0 {
34+
return errors.New("generator configs are missing. At least one is required. Please set them and try again")
35+
}
36+
37+
return nil
38+
}
39+
40+
func (b *BasicData) IsComparable(otherData BasicData) error {
41+
// are all configs present? do they have the same schedule type? do they have the same segments? is call timeout the same? is rate limit timeout the same?
42+
if len(b.GeneratorConfigs) != len(otherData.GeneratorConfigs) {
43+
return fmt.Errorf("generator configs count is different. Expected %d, got %d", len(b.GeneratorConfigs), len(otherData.GeneratorConfigs))
44+
}
45+
46+
for name1, cfg1 := range b.GeneratorConfigs {
47+
if cfg2, ok := otherData.GeneratorConfigs[name1]; !ok {
48+
return fmt.Errorf("generator config %s is missing from the other report", name1)
49+
} else {
50+
if err := compareGeneratorConfigs(cfg1, cfg2); err != nil {
51+
return err
52+
}
53+
}
54+
}
55+
56+
for name2 := range otherData.GeneratorConfigs {
57+
if _, ok := b.GeneratorConfigs[name2]; !ok {
58+
return fmt.Errorf("generator config %s is missing from the current report", name2)
59+
}
60+
}
61+
62+
// TODO: would be good to be able to check if Gun and VU are the same, but idk yet how we could do that easily [hash the code?]
63+
64+
return nil
65+
}
66+
67+
func compareGeneratorConfigs(cfg1, cfg2 *wasp.Config) error {
68+
if cfg1.LoadType != cfg2.LoadType {
69+
return fmt.Errorf("load types are different. Expected %s, got %s", cfg1.LoadType, cfg2.LoadType)
70+
}
71+
72+
if len(cfg1.Schedule) != len(cfg2.Schedule) {
73+
return fmt.Errorf("schedules are different. Expected %d, got %d", len(cfg1.Schedule), len(cfg2.Schedule))
74+
}
75+
76+
for i, segment1 := range cfg1.Schedule {
77+
segment2 := cfg2.Schedule[i]
78+
if segment1 == nil {
79+
return fmt.Errorf("schedule at index %d is nil in the current report", i)
80+
}
81+
if segment2 == nil {
82+
return fmt.Errorf("schedule at index %d is nil in the other report", i)
83+
}
84+
if *segment1 != *segment2 {
85+
return fmt.Errorf("schedules at index %d are different. Expected %s, got %s", i, mustMarshallSegment(segment1), mustMarshallSegment(segment2))
86+
}
87+
}
88+
89+
if cfg1.CallTimeout != cfg2.CallTimeout {
90+
return fmt.Errorf("call timeouts are different. Expected %s, got %s", cfg1.CallTimeout, cfg2.CallTimeout)
91+
}
92+
93+
if cfg1.RateLimitUnitDuration != cfg2.RateLimitUnitDuration {
94+
return fmt.Errorf("rate limit unit durations are different. Expected %s, got %s", cfg1.RateLimitUnitDuration, cfg2.RateLimitUnitDuration)
95+
}
96+
97+
return nil
98+
}
99+
100+
func mustMarshallSegment(segment *wasp.Segment) string {
101+
segmentBytes, err := json.MarshalIndent(segment, "", " ")
102+
if err != nil {
103+
panic(err)
104+
}
105+
106+
return string(segmentBytes)
107+
}

‎wasp/benchspy/loki.go‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
package benchspy
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/url"
7+
"reflect"
8+
"strings"
9+
"time"
10+
11+
"github.com/pkg/errors"
12+
"github.com/smartcontractkit/chainlink-testing-framework/lib/client"
13+
"github.com/smartcontractkit/chainlink-testing-framework/wasp"
14+
)
15+
16+
func NewLokiQuery(queries map[string]string, lokiConfig *wasp.LokiConfig) *LokiQuery {
17+
return &LokiQuery{
18+
Kind: "loki",
19+
Queries: queries,
20+
LokiConfig: lokiConfig,
21+
QueryResults: make(map[string][]string),
22+
}
23+
}
24+
25+
type LokiQuery struct {
26+
Kind string `json:"kind"`
27+
// Test metrics
28+
StartTime time.Time `json:"start_time"`
29+
EndTime time.Time `json:"end_time"`
30+
31+
// Performance queries
32+
// a map of name to query template, ex: "average cpu usage": "avg(rate(cpu_usage_seconds_total[5m]))"
33+
Queries map[string]string `json:"queries"`
34+
// Performance queries results
35+
// can be anything, avg RPS, amount of errors, 95th percentile of CPU utilization, etc
36+
QueryResults map[string][]string `json:"query_results"`
37+
// In case something went wrong
38+
Errors []error `json:"errors"`
39+
40+
LokiConfig *wasp.LokiConfig `json:"-"`
41+
}
42+
43+
func (l *LokiQuery) Results() map[string][]string {
44+
return l.QueryResults
45+
}
46+
47+
func (l *LokiQuery) IsComparable(otherQueryExecutor QueryExecutor) error {
48+
otherType := reflect.TypeOf(otherQueryExecutor)
49+
50+
if otherType != reflect.TypeOf(l) {
51+
return fmt.Errorf("expected type %s, got %s", reflect.TypeOf(l), otherType)
52+
}
53+
54+
return l.compareLokiQueries(otherQueryExecutor.(*LokiQuery).Queries)
55+
}
56+
57+
func (l *LokiQuery) Validate() error {
58+
if len(l.Queries) == 0 {
59+
return errors.New("there are no Loki queries, there's nothing to fetch. Please set them and try again")
60+
}
61+
if l.LokiConfig == nil {
62+
return errors.New("loki config is missing. Please set it and try again")
63+
}
64+
65+
return nil
66+
}
67+
68+
func (l *LokiQuery) Execute() error {
69+
splitAuth := strings.Split(l.LokiConfig.BasicAuth, ":")
70+
var basicAuth client.LokiBasicAuth
71+
if len(splitAuth) == 2 {
72+
basicAuth = client.LokiBasicAuth{
73+
Login: splitAuth[0],
74+
Password: splitAuth[1],
75+
}
76+
}
77+
78+
l.QueryResults = make(map[string][]string)
79+
80+
// TODO this can also be parallelized, just remember to add a mutex for writing to results map
81+
for name, query := range l.Queries {
82+
queryParams := client.LokiQueryParams{
83+
Query: query,
84+
StartTime: l.StartTime,
85+
EndTime: l.EndTime,
86+
Limit: 1000, //TODO make this configurable
87+
}
88+
89+
parsedLokiUrl, err := url.Parse(l.LokiConfig.URL)
90+
if err != nil {
91+
return errors.Wrapf(err, "failed to parse Loki URL %s", l.LokiConfig.URL)
92+
}
93+
94+
lokiUrl := parsedLokiUrl.Scheme + "://" + parsedLokiUrl.Host
95+
lokiClient := client.NewLokiClient(lokiUrl, l.LokiConfig.TenantID, basicAuth, queryParams)
96+
97+
ctx, cancelFn := context.WithTimeout(context.Background(), l.LokiConfig.Timeout)
98+
rawLogs, err := lokiClient.QueryLogs(ctx)
99+
if err != nil {
100+
l.Errors = append(l.Errors, err)
101+
cancelFn()
102+
continue
103+
}
104+
105+
cancelFn()
106+
l.QueryResults[name] = []string{}
107+
for _, log := range rawLogs {
108+
l.QueryResults[name] = append(l.QueryResults[name], log.Log)
109+
}
110+
}
111+
112+
if len(l.Errors) > 0 {
113+
return errors.New("there were errors while fetching the results. Please check the errors and try again")
114+
}
115+
116+
return nil
117+
}
118+
119+
func (l *LokiQuery) compareLokiQueries(other map[string]string) error {
120+
this := l.Queries
121+
if len(this) != len(other) {
122+
return fmt.Errorf("queries count is different. Expected %d, got %d", len(this), len(other))
123+
}
124+
125+
for name1, query1 := range this {
126+
if query2, ok := other[name1]; !ok {
127+
return fmt.Errorf("query %s is missing from the other report", name1)
128+
} else {
129+
if query1 != query2 {
130+
return fmt.Errorf("query %s is different. Expected %s, got %s", name1, query1, query2)
131+
}
132+
}
133+
}
134+
135+
for name2 := range other {
136+
if _, ok := this[name2]; !ok {
137+
return fmt.Errorf("query %s is missing from the current report", name2)
138+
}
139+
}
140+
141+
return nil
142+
}
143+
144+
func (l *LokiQuery) TimeRange(start, end time.Time) {
145+
l.StartTime = start
146+
l.EndTime = end
147+
}

‎wasp/benchspy/report.go‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package benchspy
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
)
7+
8+
// StandardReport is a report that contains all the necessary data for a performance test
9+
type StandardReport struct {
10+
BasicData
11+
LocalReportStorage
12+
ResourceReporter
13+
QueryExecutors []QueryExecutor `json:"query_executors"`
14+
}
15+
16+
func (b *StandardReport) Store() (string, error) {
17+
return b.LocalReportStorage.Store(b.TestName, b.CommitOrTag, b)
18+
}
19+
20+
func (b *StandardReport) Load() error {
21+
return b.LocalReportStorage.Load(b.TestName, b.CommitOrTag, b)
22+
}
23+
24+
func (b *StandardReport) Fetch() error {
25+
basicErr := b.BasicData.Validate()
26+
if basicErr != nil {
27+
return basicErr
28+
}
29+
30+
// TODO parallelize it
31+
for _, queryExecutor := range b.QueryExecutors {
32+
queryExecutor.TimeRange(b.TestStart, b.TestEnd)
33+
34+
if validateErr := queryExecutor.Validate(); validateErr != nil {
35+
return validateErr
36+
}
37+
38+
if execErr := queryExecutor.Execute(); execErr != nil {
39+
return execErr
40+
}
41+
}
42+
43+
resourceErr := b.FetchResources()
44+
if resourceErr != nil {
45+
return resourceErr
46+
}
47+
48+
return nil
49+
}
50+
51+
func (b *StandardReport) IsComparable(otherReport StandardReport) error {
52+
basicErr := b.BasicData.IsComparable(otherReport.BasicData)
53+
if basicErr != nil {
54+
return basicErr
55+
}
56+
57+
if resourceErr := b.CompareResources(&otherReport.ResourceReporter); resourceErr != nil {
58+
return resourceErr
59+
}
60+
61+
for i, queryExecutor := range b.QueryExecutors {
62+
queryErr := queryExecutor.IsComparable(otherReport.QueryExecutors[i])
63+
if queryErr != nil {
64+
return queryErr
65+
}
66+
}
67+
68+
return nil
69+
}
70+
71+
func (s *StandardReport) UnmarshalJSON(data []byte) error {
72+
// Define a helper struct with QueryExecutors as json.RawMessage
73+
type Alias StandardReport
74+
var raw struct {
75+
Alias
76+
QueryExecutors []json.RawMessage `json:"query_executors"`
77+
}
78+
79+
// Unmarshal into the helper struct to populate other fields automatically
80+
if err := json.Unmarshal(data, &raw); err != nil {
81+
return err
82+
}
83+
84+
var queryExecutors []QueryExecutor
85+
86+
// Manually handle QueryExecutors
87+
for _, rawExecutor := range raw.QueryExecutors {
88+
var typeIndicator struct {
89+
Kind string `json:"kind"`
90+
}
91+
if err := json.Unmarshal(rawExecutor, &typeIndicator); err != nil {
92+
return err
93+
}
94+
95+
var executor QueryExecutor
96+
switch typeIndicator.Kind {
97+
case "loki":
98+
executor = &LokiQuery{}
99+
default:
100+
return fmt.Errorf("unknown query executor type: %s", typeIndicator.Kind)
101+
}
102+
103+
if err := json.Unmarshal(rawExecutor, executor); err != nil {
104+
return err
105+
}
106+
107+
queryExecutors = append(s.QueryExecutors, executor)
108+
}
109+
110+
// Copy the automatically unmarshalled fields back to the main struct
111+
*s = StandardReport(raw.Alias)
112+
s.QueryExecutors = queryExecutors
113+
return nil
114+
}

0 commit comments

Comments
 (0)