Skip to content

Commit c2236dd

Browse files
TofelTofel
committed
add helper methods for fetching current and previous report
1 parent c2cf545 commit c2236dd

File tree

4 files changed

+146
-118
lines changed

4 files changed

+146
-118
lines changed

‎wasp/benchspy/prometheus.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ import (
1414
"github.com/smartcontractkit/chainlink-testing-framework/lib/client"
1515
)
1616

17+
type PrometheusConfig struct {
18+
Url string
19+
NameRegexPatterns []string
20+
}
21+
22+
var WithoutPrometheus *PrometheusConfig = nil
23+
1724
type PrometheusQueryExecutor struct {
1825
KindName string `json:"kind"`
1926
startTime, endTime time.Time `json:"-"`

‎wasp/benchspy/report.go‎

Lines changed: 82 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"strings"
88

99
"github.com/pkg/errors"
10+
"github.com/prometheus/common/model"
1011
"github.com/smartcontractkit/chainlink-testing-framework/wasp"
1112
"golang.org/x/sync/errgroup"
1213
)
@@ -60,6 +61,38 @@ func ResultsAs[Type any](newType Type, queryExecutors []QueryExecutor, queryExec
6061
return results, nil
6162
}
6263

64+
func MustAllLokiResults(sr *StandardReport) map[string][]string {
65+
results, err := ResultsAs([]string{}, sr.QueryExecutors, StandardQueryExecutor_Loki)
66+
if err != nil {
67+
panic(err)
68+
}
69+
return results
70+
}
71+
72+
func MustAllGeneratorResults(sr *StandardReport) map[string]string {
73+
results, err := ResultsAs("string", sr.QueryExecutors, StandardQueryExecutor_Generator)
74+
if err != nil {
75+
panic(err)
76+
}
77+
return results
78+
}
79+
80+
func MustAllPrometheusResults(sr *StandardReport) map[string]model.Value {
81+
results := make(map[string]model.Value)
82+
83+
for _, queryExecutor := range sr.QueryExecutors {
84+
if strings.EqualFold(queryExecutor.Kind(), string(StandardQueryExecutor_Prometheus)) {
85+
for queryName, result := range queryExecutor.Results() {
86+
if asValue, ok := result.(model.Value); ok {
87+
results[queryName] = asValue
88+
}
89+
}
90+
}
91+
}
92+
93+
return results
94+
}
95+
6396
func (b *StandardReport) FetchData(ctx context.Context) error {
6497
// if b.TestStart.IsZero() || b.TestEnd.IsZero() {
6598
// fillErr := b.BasicData.FillStartEndTimes()
@@ -120,17 +153,11 @@ func (b *StandardReport) IsComparable(otherReport Reporter) error {
120153
return nil
121154
}
122155

123-
type PrometheusConfig struct {
124-
Url string
125-
NameRegexPatterns []string
126-
}
127-
128-
var WithoutPrometheus *PrometheusConfig = nil
129-
130156
type standardReportConfig struct {
131157
executorType StandardQueryExecutorType
132158
generators []*wasp.Generator
133159
prometheusConfig *PrometheusConfig
160+
reportDirectory string
134161
}
135162

136163
type StandardReportOption func(*standardReportConfig)
@@ -153,6 +180,12 @@ func WithPrometheus(prometheusConfig *PrometheusConfig) StandardReportOption {
153180
}
154181
}
155182

183+
func WithReportDirectory(reportDirectory string) StandardReportOption {
184+
return func(c *standardReportConfig) {
185+
c.reportDirectory = reportDirectory
186+
}
187+
}
188+
156189
func (c *standardReportConfig) validate() error {
157190
if c.executorType == "" {
158191
return errors.New("executor type is not set")
@@ -232,10 +265,16 @@ func NewStandardReport(commitOrTag string, opts ...StandardReportOption) (*Stand
232265
}
233266
}
234267

235-
return &StandardReport{
268+
sr := &StandardReport{
236269
BasicData: *basicData,
237270
QueryExecutors: queryExecutors,
238-
}, nil
271+
}
272+
273+
if config.reportDirectory != "" {
274+
sr.LocalStorage.Directory = config.reportDirectory
275+
}
276+
277+
return sr, nil
239278
}
240279

241280
func initStandardQueryExecutor(kind StandardQueryExecutorType, basicData *BasicData, g *wasp.Generator) (QueryExecutor, error) {
@@ -399,3 +438,37 @@ func convertQueryResults(results map[string]interface{}) (map[string]interface{}
399438
}
400439
return converted, nil
401440
}
441+
442+
func FetchNewReportAndLoadLatestPrevious(ctx context.Context, newCommitOrTag string, newReportOpts ...StandardReportOption) (newReport, previousReport *StandardReport, err error) {
443+
newReport, err = NewStandardReport(newCommitOrTag, newReportOpts...)
444+
if err != nil {
445+
return
446+
}
447+
448+
config := standardReportConfig{}
449+
for _, opt := range newReportOpts {
450+
opt(&config)
451+
}
452+
453+
var localStorage LocalStorage
454+
455+
if config.reportDirectory != "" {
456+
localStorage.Directory = config.reportDirectory
457+
}
458+
459+
previousReport = &StandardReport{
460+
LocalStorage: localStorage,
461+
}
462+
463+
if err = previousReport.LoadLatest(newReport.TestName); err != nil {
464+
return
465+
}
466+
467+
if err = newReport.FetchData(ctx); err != nil {
468+
return
469+
}
470+
471+
err = newReport.IsComparable(previousReport)
472+
473+
return
474+
}

‎wasp/benchspy/storage.go‎

Lines changed: 45 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -93,52 +93,56 @@ func (l *LocalStorage) Load(testName, commitOrTag string, report interface{}) er
9393
return fmt.Errorf("no reports found in directory %s", l.Directory)
9494
}
9595

96-
// Find git root
97-
cmd := exec.Command("git", "rev-parse", "--show-toplevel")
98-
cmd.Dir = l.Directory
99-
out, err := cmd.Output()
100-
if err != nil {
101-
return errors.Wrap(err, "failed to find git root")
102-
}
103-
gitRoot := strings.TrimSpace(string(out))
104-
105-
// Resolve all refs to commit hashes
106-
resolvedRefs := make(map[string]string)
107-
for _, ref := range refs {
108-
cmd = exec.Command("git", "rev-parse", ref)
109-
cmd.Dir = gitRoot
110-
if out, err := cmd.Output(); err == nil {
111-
resolvedRefs[ref] = strings.TrimSpace(string(out))
96+
if len(refs) > 1 {
97+
// Find git root
98+
cmd := exec.Command("git", "rev-parse", "--show-toplevel")
99+
cmd.Dir = l.Directory
100+
out, err := cmd.Output()
101+
if err != nil {
102+
return errors.Wrap(err, "failed to find git root")
103+
}
104+
gitRoot := strings.TrimSpace(string(out))
105+
106+
// Resolve all refs to commit hashes
107+
resolvedRefs := make(map[string]string)
108+
for _, ref := range refs {
109+
cmd = exec.Command("git", "rev-parse", ref)
110+
cmd.Dir = gitRoot
111+
if out, err := cmd.Output(); err == nil {
112+
resolvedRefs[ref] = strings.TrimSpace(string(out))
113+
}
112114
}
113-
}
114115

115-
// Find latest among resolved commits
116-
var commitRefs []string
117-
for _, hash := range resolvedRefs {
118-
commitRefs = append(commitRefs, hash)
119-
}
116+
// Find latest among resolved commits
117+
var commitRefs []string
118+
for _, hash := range resolvedRefs {
119+
commitRefs = append(commitRefs, hash)
120+
}
120121

121-
args := append([]string{"rev-list", "--topo-order", "--date-order", "--max-count=1"}, commitRefs...)
122-
cmd = exec.Command("git", args...)
123-
cmd.Dir = gitRoot
124-
out, err = cmd.Output()
125-
if err != nil {
126-
return errors.Wrap(err, "failed to find latest reference")
127-
}
128-
latestCommit := strings.TrimSpace(string(out))
129-
130-
// Find original ref for this commit
131-
foundOriginal := false
132-
for origRef, hash := range resolvedRefs {
133-
if hash == latestCommit {
134-
ref = origRef
135-
foundOriginal = true
136-
break
122+
args := append([]string{"rev-list", "--topo-order", "--date-order", "--max-count=1"}, commitRefs...)
123+
cmd = exec.Command("git", args...)
124+
cmd.Dir = gitRoot
125+
out, err = cmd.Output()
126+
if err != nil {
127+
return errors.Wrap(err, "failed to find latest reference")
128+
}
129+
latestCommit := strings.TrimSpace(string(out))
130+
131+
// Find original ref for this commit
132+
foundOriginal := false
133+
for origRef, hash := range resolvedRefs {
134+
if hash == latestCommit {
135+
ref = origRef
136+
foundOriginal = true
137+
break
138+
}
137139
}
138-
}
139140

140-
if !foundOriginal {
141-
return fmt.Errorf("no file found for latest commit %s. This should never happen", latestCommit)
141+
if !foundOriginal {
142+
return fmt.Errorf("no file found for latest commit %s. This should never happen", latestCommit)
143+
}
144+
} else {
145+
ref = refs[0]
142146
}
143147
} else {
144148
ref = commitOrTag

‎wasp/benchspy_test.go‎

Lines changed: 12 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -181,13 +181,8 @@ func TestBenchSpyWithTwoLokiQueries(t *testing.T) {
181181
isComparableErrs := previousReport.IsComparable(&currentReport)
182182
require.Empty(t, isComparableErrs, "reports were not comparable", isComparableErrs)
183183

184-
currentAsStringSlice, castErr := benchspy.ResultsAs([]string{}, currentReport.QueryExecutors, benchspy.StandardQueryExecutor_Loki)
185-
require.NoError(t, castErr, "failed to cast results to string slice")
186-
require.NotEmpty(t, currentAsStringSlice, "results were empty")
187-
188-
previousAsStringSlice, castErr := benchspy.ResultsAs([]string{}, previousReport.QueryExecutors, benchspy.StandardQueryExecutor_Loki)
189-
require.NoError(t, castErr, "failed to cast results to string slice")
190-
require.NotEmpty(t, previousAsStringSlice, "results were empty")
184+
currentAsStringSlice := benchspy.MustAllLokiResults(&currentReport)
185+
previousAsStringSlice := benchspy.MustAllLokiResults(&previousReport)
191186

192187
// vu over time
193188
require.NotEmpty(t, currentReport.QueryExecutors[0].Results()["vu_over_time"], "vu_over_time results were missing from current report")
@@ -271,43 +266,21 @@ func TestBenchSpyWithStandardLokiMetrics(t *testing.T) {
271266

272267
gen.Run(true)
273268

274-
currentReport, err := benchspy.NewStandardReport("e7fc5826a572c09f8b93df3b9f674113372ce925", benchspy.WithStandardQueryExecutorType(benchspy.StandardQueryExecutor_Loki), benchspy.WithGenerators(gen))
275-
require.NoError(t, err)
276-
277269
fetchCtx, cancelFn := context.WithTimeout(context.Background(), 60*time.Second)
278270
defer cancelFn()
279271

280-
fetchErr := currentReport.FetchData(fetchCtx)
281-
require.NoError(t, fetchErr, "failed to fetch current report")
282-
283-
path, storeErr := currentReport.Store()
284-
require.NoError(t, storeErr, "failed to store current report", path)
285-
286-
// this is only needed, because we are using a non-standard directory
287-
// otherwise, the Load method would be able to find the file
288-
previousReport := benchspy.StandardReport{
289-
LocalStorage: benchspy.LocalStorage{
290-
Directory: "test_performance_reports",
291-
},
292-
}
293-
loadErr := previousReport.Load(t.Name(), "e7fc5826a572c09f8b93df3b9f674113372ce924")
294-
require.NoError(t, loadErr, "failed to load previous report")
295-
296-
isComparableErrs := previousReport.IsComparable(currentReport)
297-
require.Empty(t, isComparableErrs, "reports were not comparable", isComparableErrs)
272+
currentReport, previousReport, err := benchspy.FetchNewReportAndLoadLatestPrevious(fetchCtx, "e7fc5826a572c09f8b93df3b9f674113372ce925", benchspy.WithStandardQueryExecutorType(benchspy.StandardQueryExecutor_Loki), benchspy.WithGenerators(gen), benchspy.WithReportDirectory("test_performance_reports"))
273+
require.NoError(t, err, "failed to fetch current report or load the previous one")
298274

299-
currentAsStringSlice, castErr := benchspy.ResultsAs([]string{}, currentReport.QueryExecutors, benchspy.StandardQueryExecutor_Loki)
300-
require.NoError(t, castErr, "failed to cast results to string slice")
301-
require.NotEmpty(t, currentAsStringSlice, "results were empty")
275+
// path, storeErr := currentReport.Store()
276+
// require.NoError(t, storeErr, "failed to store current report", path)
302277

303-
previousAsStringSlice, castErr := benchspy.ResultsAs([]string{}, previousReport.QueryExecutors, benchspy.StandardQueryExecutor_Loki)
304-
require.NoError(t, castErr, "failed to cast results to string slice")
305-
require.NotEmpty(t, previousAsStringSlice, "results were empty")
278+
currentAsStringSlice := benchspy.MustAllLokiResults(currentReport)
279+
previousAsStringSlice := benchspy.MustAllLokiResults(previousReport)
306280

307281
var compareMedian = func(metricName benchspy.StandardLoadMetric) {
308282
require.NotEmpty(t, currentAsStringSlice[string(metricName)], "%s results were missing from current report", string(metricName))
309283
require.NotEmpty(t, previousAsStringSlice[string(metricName)], "%s results were missing from previous report", string(metricName))
310-
require.Equal(t, len(currentAsStringSlice[string(metricName)]), len(previousAsStringSlice[string(metricName)]), "%s results are not the same length", string(metricName))
311284

312285
currentFloatSlice, err := benchspy.StringSliceToFloat64Slice(currentAsStringSlice[string(metricName)])
313286
require.NoError(t, err, "failed to convert %s results to float64 slice", string(metricName))
@@ -375,13 +348,8 @@ func TestBenchSpyWithStandardGeneratorMetrics(t *testing.T) {
375348
isComparableErrs := previousReport.IsComparable(currentReport)
376349
require.Empty(t, isComparableErrs, "reports were not comparable", isComparableErrs)
377350

378-
currentAsString, castErr := benchspy.ResultsAs("", currentReport.QueryExecutors, benchspy.StandardQueryExecutor_Generator)
379-
require.NoError(t, castErr, "failed to cast results to string slice")
380-
require.NotEmpty(t, currentAsString, "results were empty")
381-
382-
previousAsString, castErr := benchspy.ResultsAs("", previousReport.QueryExecutors, benchspy.StandardQueryExecutor_Generator)
383-
require.NoError(t, castErr, "failed to cast results to string slice")
384-
require.NotEmpty(t, previousAsString, "results were empty")
351+
currentAsString := benchspy.MustAllGeneratorResults(currentReport)
352+
previousAsString := benchspy.MustAllGeneratorResults(&previousReport)
385353

386354
var compareValues = func(metricName benchspy.StandardLoadMetric) {
387355
require.NotEmpty(t, currentAsString[string(metricName)], "%s results were missing from current report", string(metricName))
@@ -407,30 +375,6 @@ func TestBenchSpyWithStandardGeneratorMetrics(t *testing.T) {
407375
compareValues(benchspy.ErrorRate)
408376
}
409377

410-
// func TestBenchSpy_Prometheus(t *testing.T) {
411-
// t.Skip("skipping test, since it requires a running CTFv2 node_set and observability stack [start it manually and run the test]")
412-
413-
// before := time.Now().Add(-5 * time.Minute)
414-
// // exclude bootstrap node
415-
// prometheusNodeReporter, err := benchspy.NewStandardPrometheusQueryExecutor("http://localhost:9090", before, time.Now(), `node[^0]`)
416-
// require.NoError(t, err)
417-
418-
// fetchErr := prometheusNodeReporter.Execute(context.Background())
419-
// require.NoError(t, fetchErr, "failed to fetch prometheus node resources")
420-
421-
// resourcesAsValue := prometheusNodeReporter.MustResultsAsValue()
422-
// medianCpuUsagePerNode := resourcesAsValue[string(benchspy.MedianCPUUsage)]
423-
// require.Equal(t, medianCpuUsagePerNode.Type(), model.ValVector, "median cpu usage per node should be a vector")
424-
425-
// medianCpuUsagePerNodeVector := medianCpuUsagePerNode.(model.Vector)
426-
// require.NotEmpty(t, medianCpuUsagePerNodeVector, "median cpu usage per node vector should not be empty")
427-
// require.Equal(t, 4, len(medianCpuUsagePerNodeVector), "median cpu usage per node vector should have 4 elements")
428-
429-
// for _, sample := range medianCpuUsagePerNodeVector {
430-
// require.NotZero(t, sample.Value, "median cpu usage per node should not be zero")
431-
// }
432-
// }
433-
434378
func TestBenchSpy_Prometheus_And_Generator(t *testing.T) {
435379
// this test requires CTFv2 node_set with observability stack to be running
436380
previousReport := benchspy.StandardReport{
@@ -473,8 +417,8 @@ func TestBenchSpy_Prometheus_And_Generator(t *testing.T) {
473417
isComparableErrs := previousReport.IsComparable(currentReport)
474418
require.Empty(t, isComparableErrs, "reports were not comparable", isComparableErrs)
475419

476-
currentAsValues := currentReport.QueryExecutors[1].(*benchspy.PrometheusQueryExecutor).MustResultsAsValue()
477-
previousAsValues := previousReport.QueryExecutors[1].(*benchspy.PrometheusQueryExecutor).MustResultsAsValue()
420+
currentAsValues := benchspy.MustAllPrometheusResults(currentReport)
421+
previousAsValues := benchspy.MustAllPrometheusResults(&previousReport)
478422

479423
assert.Equal(t, len(currentAsValues), len(previousAsValues), "number of metrics in results should be the same")
480424

0 commit comments

Comments
 (0)