Skip to content

Commit 305088e

Browse files
TofelTofel
committed
more unit tests
1 parent c2236dd commit 305088e

File tree

6 files changed

+303
-23
lines changed

6 files changed

+303
-23
lines changed

‎wasp/benchspy/generator_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ func TestBenchSpy_GeneratorQueryExecutor_Execute(t *testing.T) {
214214
cfg.Gun = fakeGun
215215

216216
gen, err := wasp.NewGenerator(cfg)
217-
assert.NoError(t, err)
217+
require.NoError(t, err)
218218

219219
gen.Run(true)
220220

‎wasp/benchspy/prometheus.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,8 @@ func (r *PrometheusQueryExecutor) MustResultsAsValue() map[string]model.Value {
146146
}
147147

148148
func (r *PrometheusQueryExecutor) TimeRange(startTime, endTime time.Time) {
149-
// not sure if we need to set the time range for Prometheus
150-
// I think we should remove that method all together from all interfaces
151-
// and instead make sure that all segments have start/end times (i.e. they were executed), when new report is created
149+
r.startTime = startTime
150+
r.endTime = endTime
152151
return
153152
}
154153

‎wasp/benchspy/report.go‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -439,10 +439,10 @@ func convertQueryResults(results map[string]interface{}) (map[string]interface{}
439439
return converted, nil
440440
}
441441

442-
func FetchNewReportAndLoadLatestPrevious(ctx context.Context, newCommitOrTag string, newReportOpts ...StandardReportOption) (newReport, previousReport *StandardReport, err error) {
443-
newReport, err = NewStandardReport(newCommitOrTag, newReportOpts...)
442+
func FetchNewReportAndLoadLatestPrevious(ctx context.Context, newCommitOrTag string, newReportOpts ...StandardReportOption) (*StandardReport, *StandardReport, error) {
443+
newReport, err := NewStandardReport(newCommitOrTag, newReportOpts...)
444444
if err != nil {
445-
return
445+
return nil, nil, errors.Wrapf(err, "failed to create new report for commit or tag %s", newCommitOrTag)
446446
}
447447

448448
config := standardReportConfig{}
@@ -456,19 +456,22 @@ func FetchNewReportAndLoadLatestPrevious(ctx context.Context, newCommitOrTag str
456456
localStorage.Directory = config.reportDirectory
457457
}
458458

459-
previousReport = &StandardReport{
459+
previousReport := &StandardReport{
460460
LocalStorage: localStorage,
461461
}
462462

463463
if err = previousReport.LoadLatest(newReport.TestName); err != nil {
464-
return
464+
return nil, nil, errors.Wrapf(err, "failed to load latest report for test %s", newReport.TestName)
465465
}
466466

467467
if err = newReport.FetchData(ctx); err != nil {
468-
return
468+
return nil, nil, errors.Wrapf(err, "failed to fetch data for new report")
469469
}
470470

471471
err = newReport.IsComparable(previousReport)
472+
if err != nil {
473+
return nil, nil, errors.Wrapf(err, "new report is not comparable to previous report")
474+
}
472475

473-
return
476+
return newReport, previousReport, nil
474477
}

‎wasp/benchspy/report_test.go‎

Lines changed: 247 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -413,7 +413,7 @@ func TestBenchSpy_StandardReport_UnmarshalJSON(t *testing.T) {
413413
assert.Equal(t, "1", asStringSlice["test generator query"])
414414
})
415415

416-
t.Run("valid prometheus executor", func(t *testing.T) {
416+
t.Run("valid prometheus executor (vector)", func(t *testing.T) {
417417
jsonData := `{
418418
"test_name": "test1",
419419
"commit_or_tag": "abc123",
@@ -819,3 +819,249 @@ func TestBenchSpy_ConvertQueryResults(t *testing.T) {
819819
})
820820
})
821821
}
822+
823+
func TestBenchSpy_MustAllResults(t *testing.T) {
824+
t.Run("MustAllLokiResults", func(t *testing.T) {
825+
mockLokiExec := &MockQueryExecutor{
826+
ResultsFn: func() map[string]interface{} {
827+
return map[string]interface{}{
828+
"query1": []string{"log1", "log2"},
829+
"query2": []string{"log3", "log4"},
830+
}
831+
},
832+
KindFn: func() string {
833+
return string(StandardQueryExecutor_Loki)
834+
},
835+
}
836+
837+
sr := &StandardReport{
838+
QueryExecutors: []QueryExecutor{mockLokiExec},
839+
}
840+
841+
results := MustAllLokiResults(sr)
842+
assert.Equal(t, 2, len(results))
843+
assert.Equal(t, []string{"log1", "log2"}, results["query1"])
844+
assert.Equal(t, []string{"log3", "log4"}, results["query2"])
845+
})
846+
847+
t.Run("MustAllGeneratorResults", func(t *testing.T) {
848+
mockGenExec := &MockQueryExecutor{
849+
ResultsFn: func() map[string]interface{} {
850+
return map[string]interface{}{
851+
"query1": "result1",
852+
"query2": "result2",
853+
}
854+
},
855+
KindFn: func() string {
856+
return string(StandardQueryExecutor_Generator)
857+
},
858+
}
859+
860+
sr := &StandardReport{
861+
QueryExecutors: []QueryExecutor{mockGenExec},
862+
}
863+
864+
results := MustAllGeneratorResults(sr)
865+
assert.Equal(t, 2, len(results))
866+
assert.Equal(t, "result1", results["query1"])
867+
assert.Equal(t, "result2", results["query2"])
868+
})
869+
870+
t.Run("MustAllPrometheusResults", func(t *testing.T) {
871+
vector1 := model.Vector{
872+
&model.Sample{
873+
Value: 1.23,
874+
},
875+
}
876+
vector2 := model.Vector{
877+
&model.Sample{
878+
Value: 4.56,
879+
},
880+
}
881+
882+
mockPromExec := &MockQueryExecutor{
883+
ResultsFn: func() map[string]interface{} {
884+
return map[string]interface{}{
885+
"query1": vector1,
886+
"query2": vector2,
887+
}
888+
},
889+
KindFn: func() string {
890+
return string(StandardQueryExecutor_Prometheus)
891+
},
892+
}
893+
894+
sr := &StandardReport{
895+
QueryExecutors: []QueryExecutor{mockPromExec},
896+
}
897+
898+
results := MustAllPrometheusResults(sr)
899+
assert.Equal(t, 2, len(results))
900+
assert.Equal(t, model.Vector(vector1), results["query1"].(model.Vector))
901+
assert.Equal(t, model.Vector(vector2), results["query2"].(model.Vector))
902+
})
903+
904+
t.Run("MustAllLokiResults panics on wrong type", func(t *testing.T) {
905+
mockExec := &MockQueryExecutor{
906+
ResultsFn: func() map[string]interface{} {
907+
return map[string]interface{}{
908+
"query1": 123, // wrong type
909+
}
910+
},
911+
KindFn: func() string {
912+
return string(StandardQueryExecutor_Loki)
913+
},
914+
}
915+
916+
sr := &StandardReport{
917+
QueryExecutors: []QueryExecutor{mockExec},
918+
}
919+
920+
assert.Panics(t, func() {
921+
MustAllLokiResults(sr)
922+
})
923+
})
924+
925+
t.Run("MustAllGeneratorResults panics on wrong type", func(t *testing.T) {
926+
mockExec := &MockQueryExecutor{
927+
ResultsFn: func() map[string]interface{} {
928+
return map[string]interface{}{
929+
"query1": []string{"wrong", "type"},
930+
}
931+
},
932+
KindFn: func() string {
933+
return string(StandardQueryExecutor_Generator)
934+
},
935+
}
936+
937+
sr := &StandardReport{
938+
QueryExecutors: []QueryExecutor{mockExec},
939+
}
940+
941+
assert.Panics(t, func() {
942+
MustAllGeneratorResults(sr)
943+
})
944+
})
945+
946+
t.Run("Results from mixed executors", func(t *testing.T) {
947+
mockLokiExec := &MockQueryExecutor{
948+
ResultsFn: func() map[string]interface{} {
949+
return map[string]interface{}{
950+
"loki_query": []string{"log1", "log2"},
951+
}
952+
},
953+
KindFn: func() string {
954+
return string(StandardQueryExecutor_Loki)
955+
},
956+
}
957+
958+
mockGenExec := &MockQueryExecutor{
959+
ResultsFn: func() map[string]interface{} {
960+
return map[string]interface{}{
961+
"gen_query": "result1",
962+
}
963+
},
964+
KindFn: func() string {
965+
return string(StandardQueryExecutor_Generator)
966+
},
967+
}
968+
969+
sr := &StandardReport{
970+
QueryExecutors: []QueryExecutor{mockLokiExec, mockGenExec},
971+
}
972+
973+
lokiResults := MustAllLokiResults(sr)
974+
assert.Equal(t, 1, len(lokiResults))
975+
assert.Equal(t, []string{"log1", "log2"}, lokiResults["loki_query"])
976+
977+
genResults := MustAllGeneratorResults(sr)
978+
assert.Equal(t, 1, len(genResults))
979+
assert.Equal(t, "result1", genResults["gen_query"])
980+
})
981+
}
982+
983+
func TestBenchSpy_FetchNewReportAndLoadLatestPrevious(t *testing.T) {
984+
baseTime := time.Now()
985+
basicGen := &wasp.Generator{
986+
Cfg: &wasp.Config{
987+
T: t,
988+
GenName: "test-gen",
989+
Labels: map[string]string{
990+
"branch": "main",
991+
"commit": "abc123",
992+
},
993+
Schedule: []*wasp.Segment{
994+
{StartTime: baseTime, EndTime: baseTime.Add(time.Hour), From: 1, Duration: 2 * time.Second},
995+
},
996+
LokiConfig: lokiConfig,
997+
},
998+
}
999+
1000+
t.Run("successful execution", func(t *testing.T) {
1001+
tmpDir := t.TempDir()
1002+
1003+
cfg := &wasp.Config{
1004+
T: t,
1005+
LoadType: wasp.RPS,
1006+
GenName: "test-gen",
1007+
Labels: map[string]string{
1008+
"branch": "main",
1009+
"commit": "abc123",
1010+
},
1011+
Schedule: []*wasp.Segment{
1012+
{StartTime: baseTime, EndTime: baseTime.Add(time.Hour), From: 1, Duration: 2 * time.Second, Type: wasp.SegmentType_Plain},
1013+
},
1014+
}
1015+
1016+
fakeGun := &fakeGun{
1017+
maxSuccesses: 4,
1018+
maxFailures: 3,
1019+
schedule: cfg.Schedule[0],
1020+
}
1021+
1022+
cfg.Gun = fakeGun
1023+
1024+
gen, err := wasp.NewGenerator(cfg)
1025+
require.NoError(t, err)
1026+
1027+
gen.Run(true)
1028+
1029+
prevReport, err := NewStandardReport("a7fc5826a572c09f8b93df3b9f674113372ce924",
1030+
WithStandardQueryExecutorType(StandardQueryExecutor_Generator),
1031+
WithGenerators(gen),
1032+
WithReportDirectory(tmpDir))
1033+
require.NoError(t, err)
1034+
_, err = prevReport.Store()
1035+
require.NoError(t, err)
1036+
1037+
newReport, prevLoadedReport, err := FetchNewReportAndLoadLatestPrevious(
1038+
context.Background(),
1039+
"new-commit",
1040+
WithStandardQueryExecutorType(StandardQueryExecutor_Generator),
1041+
WithGenerators(gen),
1042+
WithReportDirectory(tmpDir),
1043+
)
1044+
require.NoError(t, err)
1045+
1046+
assert.NotNil(t, newReport)
1047+
assert.NotNil(t, prevLoadedReport)
1048+
assert.Equal(t, "a7fc5826a572c09f8b93df3b9f674113372ce924", prevLoadedReport.CommitOrTag)
1049+
})
1050+
1051+
t.Run("no previous report", func(t *testing.T) {
1052+
tmpDir := t.TempDir()
1053+
1054+
basicGen.Cfg.T = t
1055+
newReport, prevReport, err := FetchNewReportAndLoadLatestPrevious(
1056+
context.Background(),
1057+
"new-commit-7",
1058+
WithStandardQueryExecutorType(StandardQueryExecutor_Generator),
1059+
WithGenerators(basicGen),
1060+
WithReportDirectory(tmpDir),
1061+
)
1062+
1063+
assert.Error(t, err)
1064+
assert.Nil(t, newReport)
1065+
assert.Nil(t, prevReport)
1066+
})
1067+
}

‎wasp/benchspy/storage.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ func (l *LocalStorage) defaultDirectoryIfEmpty() {
2626
}
2727
}
2828

29+
func (l *LocalStorage) cleanTestName(testName string) string {
30+
// nested tests might contain slashes, replace them with underscores
31+
return strings.ReplaceAll(testName, "/", "_")
32+
}
33+
2934
func (l *LocalStorage) Store(testName, commitOrTag string, report interface{}) (string, error) {
3035
l.defaultDirectoryIfEmpty()
3136
asJson, err := json.MarshalIndent(report, "", " ")
@@ -39,7 +44,8 @@ func (l *LocalStorage) Store(testName, commitOrTag string, report interface{}) (
3944
}
4045
}
4146

42-
reportFilePath := filepath.Join(l.Directory, fmt.Sprintf("%s-%s.json", testName, commitOrTag))
47+
cleanTestName := l.cleanTestName(testName)
48+
reportFilePath := filepath.Join(l.Directory, fmt.Sprintf("%s-%s.json", cleanTestName, commitOrTag))
4349
reportFile, err := os.Create(reportFilePath)
4450
if err != nil {
4551
return "", errors.Wrapf(err, "failed to create file %s", reportFilePath)
@@ -66,6 +72,8 @@ func (l *LocalStorage) Load(testName, commitOrTag string, report interface{}) er
6672
return errors.New("test name is empty. Please set it and try again")
6773
}
6874

75+
cleanTestName := l.cleanTestName(testName)
76+
6977
var ref string
7078
if commitOrTag == "" {
7179
entries, err := os.ReadDir(l.Directory)
@@ -77,14 +85,14 @@ func (l *LocalStorage) Load(testName, commitOrTag string, report interface{}) er
7785
var refs []string
7886
filesByRef := make(map[string]string)
7987
for _, entry := range entries {
80-
if !entry.IsDir() && strings.Contains(entry.Name(), testName) {
88+
if !entry.IsDir() && strings.Contains(entry.Name(), cleanTestName) {
8189
parts := strings.Split(entry.Name(), "-")
8290
if len(parts) == 2 {
8391
ref := strings.TrimSuffix(parts[len(parts)-1], ".json")
8492
refs = append(refs, ref)
8593
filesByRef[ref] = filepath.Join(l.Directory, entry.Name())
8694
} else {
87-
return errors.Errorf("invalid file name: %s. Expected: %s-<ref>.json", entry.Name(), testName)
95+
return errors.Errorf("invalid file name: %s. Expected: %s-<ref>.json", entry.Name(), cleanTestName)
8896
}
8997
}
9098
}
@@ -148,7 +156,7 @@ func (l *LocalStorage) Load(testName, commitOrTag string, report interface{}) er
148156
ref = commitOrTag
149157
}
150158

151-
reportFilePath := filepath.Join(l.Directory, fmt.Sprintf("%s-%s.json", testName, ref))
159+
reportFilePath := filepath.Join(l.Directory, fmt.Sprintf("%s-%s.json", cleanTestName, ref))
152160

153161
reportFile, err := os.Open(reportFilePath)
154162
if err != nil {

0 commit comments

Comments
 (0)