diff --git a/book/src/libs/wasp/benchspy/loki_std.md b/book/src/libs/wasp/benchspy/loki_std.md index 51fa91211..21d6de1a9 100644 --- a/book/src/libs/wasp/benchspy/loki_std.md +++ b/book/src/libs/wasp/benchspy/loki_std.md @@ -112,14 +112,9 @@ var compareAverages = func(t *testing.T, metricName string, currentAsStringSlice assert.LessOrEqual(t, math.Abs(diffPrecentage), maxPrecentageDiff, "%s medians are more than 1% different", metricName, fmt.Sprintf("%.4f", diffPrecentage)) } -compareAverages( - t, - string(benchspy.MedianLatency), - currentAsStringSlice, - previousAsStringSlice, - 1.0, -) +compareAverages(t, string(benchspy.MedianLatency), currentAsStringSlice, previousAsStringSlice, 1.0) compareAverages(t, string(benchspy.Percentile95Latency), currentAsStringSlice, previousAsStringSlice, 1.0) +compareAverages(t, string(benchspy.Percentile99Latency), currentAsStringSlice, previousAsStringSlice, 1.0) compareAverages(t, string(benchspy.MaxLatency), currentAsStringSlice, previousAsStringSlice, 1.0) compareAverages(t, string(benchspy.ErrorRate), currentAsStringSlice, previousAsStringSlice, 1.0) ``` diff --git a/book/src/libs/wasp/benchspy/real_world.md b/book/src/libs/wasp/benchspy/real_world.md index 1d75e9160..ccb2c540d 100644 --- a/book/src/libs/wasp/benchspy/real_world.md +++ b/book/src/libs/wasp/benchspy/real_world.md @@ -90,6 +90,7 @@ Let’s assume you want to ensure that none of the performance metrics degrade b hasFailed, error := benchspy.CompareDirectWithThresholds( 1.0, // Max 1% worse median latency 1.0, // Max 1% worse p95 latency + 1.0, // Max 1% worse p99 latency 1.0, // Max 1% worse maximum latency 0.0, // No increase in error rate currentReport, previousReport) diff --git a/book/src/libs/wasp/benchspy/reports/standard_report.md b/book/src/libs/wasp/benchspy/reports/standard_report.md index b716ffdda..468d979f3 100644 --- a/book/src/libs/wasp/benchspy/reports/standard_report.md +++ b/book/src/libs/wasp/benchspy/reports/standard_report.md @@ -18,6 +18,7 @@ Both query executors focus on the characteristics of the load generated by `WASP Predefined metrics for both include: - Median latency - 95th percentile latency +- 99th percentile latency - Max latency - Error rate diff --git a/book/src/libs/wasp/benchspy/simplest_metrics.md b/book/src/libs/wasp/benchspy/simplest_metrics.md index 4669d2b28..a260b140d 100644 --- a/book/src/libs/wasp/benchspy/simplest_metrics.md +++ b/book/src/libs/wasp/benchspy/simplest_metrics.md @@ -15,6 +15,7 @@ hasErrors, errors := benchspy.CompareDirectWithThresholds( // maximum differences in percentages for: 1.0, // median latency 1.0, // p95 latency + 1.0, // p99 latency 1.0, // max latency 1.0, // error rate currentReport, @@ -29,6 +30,7 @@ If there are errors they will be returned as `map[string][]errors`, where key is > Both `Direct` and `Loki` query executors support following standard performance metrics out of the box: > - `median_latency` > - `p95_latency` +> - `p99_latency` > - `max_latency` > - `error_rate` @@ -43,6 +45,8 @@ Generator: vu1 +-------------------------+---------+---------+---------+ | 95th_percentile_latency | 50.7387 | 50.7622 | 0.0463 | +-------------------------+---------+---------+---------+ +| 99th_percentile_latency | 54.8192 | 51.0124 | -7.4624 | ++-------------------------+---------+---------+---------+ | max_latency | 55.7195 | 51.7248 | -7.1692 | +-------------------------+---------+---------+---------+ | error_rate | 0.0000 | 0.0000 | 0.0000 | diff --git a/wasp/.changeset/v1.51.1.md b/wasp/.changeset/v1.51.1.md new file mode 100644 index 000000000..fed6ac033 --- /dev/null +++ b/wasp/.changeset/v1.51.1.md @@ -0,0 +1 @@ +- Add p99 metric to BenchSpy's Direct and Loki standard Query Executors \ No newline at end of file diff --git a/wasp/benchspy/direct.go b/wasp/benchspy/direct.go index 12cc4bd7a..32fa5852c 100644 --- a/wasp/benchspy/direct.go +++ b/wasp/benchspy/direct.go @@ -269,6 +269,11 @@ func (dqe *DirectQueryExecutor) standardQuery(standardMetric StandardLoadMetric) return stats.Percentile(responsesToDurationFn(responses), 95) } return p95Fn, nil + case Percentile99Latency: + p99Fn := func(responses *wasp.SliceBuffer[*wasp.Response]) (float64, error) { + return stats.Percentile(responsesToDurationFn(responses), 99) + } + return p99Fn, nil case MaxLatency: maxFn := func(responses *wasp.SliceBuffer[*wasp.Response]) (float64, error) { return stats.Max(responsesToDurationFn(responses)) diff --git a/wasp/benchspy/direct_test.go b/wasp/benchspy/direct_test.go index 9d910fb84..4f91a2b6f 100644 --- a/wasp/benchspy/direct_test.go +++ b/wasp/benchspy/direct_test.go @@ -235,13 +235,15 @@ func TestBenchSpy_DirectQueryExecutor_Execute(t *testing.T) { // 4 responses with ~150ms latency (150ms sleep + some execution overhead) // and 2-3 responses with ~200ms latency (200ms sleep + some execution overhead) // expected median latency: (150ms, 151ms> - resultsAsFloats, err := ResultsAs(0.0, executor, string(MedianLatency), string(Percentile95Latency), string(ErrorRate)) + resultsAsFloats, err := ResultsAs(0.0, executor, string(MedianLatency), string(Percentile95Latency), string(Percentile99Latency), string(ErrorRate)) assert.NoError(t, err) - require.Equal(t, 3, len(resultsAsFloats)) + require.Equal(t, 4, len(resultsAsFloats)) require.InDelta(t, 151.0, resultsAsFloats[string(MedianLatency)], 1.0) // since we have 2-3 responses with 200-201ms latency, the 95th percentile should be (200ms, 201ms> require.InDelta(t, 201.0, resultsAsFloats[string(Percentile95Latency)], 1.0) + // since we have 2-3 responses with 200-201ms latency, the 99th percentile should be (199ms, 203ms> + require.InDelta(t, 201.0, resultsAsFloats[string(Percentile99Latency)], 2.0) errorRate, exists := resultsAsFloats[string(ErrorRate)] assert.True(t, exists) diff --git a/wasp/benchspy/loki.go b/wasp/benchspy/loki.go index 71b1c006d..75db885de 100644 --- a/wasp/benchspy/loki.go +++ b/wasp/benchspy/loki.go @@ -22,6 +22,7 @@ import ( var ( Loki_MedianQuery = `quantile_over_time(0.5, {branch=~"%s", commit=~"%s", go_test_name=~"%s", test_data_type=~"responses", gen_name=~"%s"} | json| unwrap duration [10s]) by (go_test_name, gen_name) / 1e6` Loki_95thQuery = `quantile_over_time(0.95, {branch=~"%s", commit=~"%s", go_test_name=~"%s", test_data_type=~"responses", gen_name=~"%s"} | json| unwrap duration [10s]) by (go_test_name, gen_name) / 1e6` + Loki_99thQuery = `quantile_over_time(0.99, {branch=~"%s", commit=~"%s", go_test_name=~"%s", test_data_type=~"responses", gen_name=~"%s"} | json| unwrap duration [10s]) by (go_test_name, gen_name) / 1e6` Loki_MaxQuery = `max(max_over_time({branch=~"%s", commit=~"%s", go_test_name=~"%s", test_data_type=~"responses", gen_name=~"%s"} | json| unwrap duration [10s]) by (go_test_name, gen_name) / 1e6)` Loki_ErrorRate = `sum(max_over_time({branch=~"%s", commit=~"%s", go_test_name=~"%s", test_data_type=~"stats", gen_name=~"%s"} | json| unwrap failed [%s]) by (node_id, go_test_name, gen_name)) by (__stream_shard__)` ) @@ -311,6 +312,8 @@ func (l *LokiQueryExecutor) standardQuery(standardMetric StandardLoadMetric, tes return fmt.Sprintf(Loki_MedianQuery, branch, commit, testName, generatorName), nil case Percentile95Latency: return fmt.Sprintf(Loki_95thQuery, branch, commit, testName, generatorName), nil + case Percentile99Latency: + return fmt.Sprintf(Loki_99thQuery, branch, commit, testName, generatorName), nil case MaxLatency: return fmt.Sprintf(Loki_MaxQuery, branch, commit, testName, generatorName), nil case ErrorRate: diff --git a/wasp/benchspy/report.go b/wasp/benchspy/report.go index c4705178d..5645f6513 100644 --- a/wasp/benchspy/report.go +++ b/wasp/benchspy/report.go @@ -168,7 +168,7 @@ func calculateDiffPercentage(current, previous float64) float64 { // CompareDirectWithThresholds evaluates the current and previous reports against specified thresholds. // It checks for significant differences in metrics and returns any discrepancies found, aiding in performance analysis. -func CompareDirectWithThresholds(medianThreshold, p95Threshold, maxThreshold, errorRateThreshold float64, currentReport, previousReport *StandardReport) (bool, error) { +func CompareDirectWithThresholds(medianThreshold, p95Threshold, p99Threshold, maxThreshold, errorRateThreshold float64, currentReport, previousReport *StandardReport) (bool, error) { if currentReport == nil || previousReport == nil { return true, errors.New("one or both reports are nil") } @@ -178,11 +178,12 @@ func CompareDirectWithThresholds(medianThreshold, p95Threshold, maxThreshold, er Str("Previous report", previousReport.CommitOrTag). Float64("Median threshold", medianThreshold). Float64("P95 threshold", p95Threshold). + Float64("P99 threshold", p99Threshold). Float64("Max threshold", maxThreshold). Float64("Error rate threshold", errorRateThreshold). Msg("Comparing Direct metrics with thresholds") - if thresholdsErr := validateThresholds(medianThreshold, p95Threshold, maxThreshold, errorRateThreshold); thresholdsErr != nil { + if thresholdsErr := validateThresholds(medianThreshold, p95Threshold, p99Threshold, maxThreshold, errorRateThreshold); thresholdsErr != nil { return true, thresholdsErr } @@ -234,6 +235,10 @@ func CompareDirectWithThresholds(medianThreshold, p95Threshold, maxThreshold, er errors[genCfg.GenName] = append(errors[genCfg.GenName], err) } + if err := compareValues(string(Percentile99Latency), genCfg.GenName, p99Threshold); err != nil { + errors[genCfg.GenName] = append(errors[genCfg.GenName], err) + } + if err := compareValues(string(MaxLatency), genCfg.GenName, maxThreshold); err != nil { errors[genCfg.GenName] = append(errors[genCfg.GenName], err) } @@ -264,7 +269,7 @@ func concatenateGeneratorErrors(errors map[string][]error) error { return goerrors.Join(errs...) } -func validateThresholds(medianThreshold, p95Threshold, maxThreshold, errorRateThreshold float64) error { +func validateThresholds(medianThreshold, p95Threshold, p99Threshold, maxThreshold, errorRateThreshold float64) error { var errs []error var validateThreshold = func(name string, threshold float64) error { @@ -282,6 +287,10 @@ func validateThresholds(medianThreshold, p95Threshold, maxThreshold, errorRateTh errs = append(errs, err) } + if err := validateThreshold("p99", p99Threshold); err != nil { + errs = append(errs, err) + } + if err := validateThreshold("max", maxThreshold); err != nil { errs = append(errs, err) } diff --git a/wasp/benchspy/report_test.go b/wasp/benchspy/report_test.go index 1aaeedf10..e69402f88 100644 --- a/wasp/benchspy/report_test.go +++ b/wasp/benchspy/report_test.go @@ -1298,6 +1298,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 100.0, string(Percentile95Latency): 200.0, + string(Percentile99Latency): 280.0, string(MaxLatency): 300.0, string(ErrorRate): 1.0, } @@ -1322,6 +1323,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 105.0, // 5% increase string(Percentile95Latency): 210.0, // 5% increase + string(Percentile99Latency): 294.0, // 5% increase string(MaxLatency): 315.0, // 5% increase string(ErrorRate): 1.05, // 5% increase } @@ -1331,7 +1333,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, errs := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + failed, errs := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) assert.False(t, failed) assert.Empty(t, errs) }) @@ -1352,6 +1354,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 100.0, string(Percentile95Latency): 200.0, + string(Percentile99Latency): 280.0, string(MaxLatency): 300.0, string(ErrorRate): 1.0, } @@ -1376,6 +1379,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 150.0, // 50% increase string(Percentile95Latency): 200.0, // no increase + string(Percentile99Latency): 280.0, // no increase string(MaxLatency): 300.0, // no increase string(ErrorRate): 1.0, // no increase } @@ -1385,7 +1389,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(10.0, 1.0, 1.0, 1.0, currentReport, previousReport) + failed, err := CompareDirectWithThresholds(10.0, 1.0, 1.0, 1.0, 1.0, currentReport, previousReport) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 50.0000%% different, which is higher than the threshold", string(MedianLatency))) @@ -1407,6 +1411,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 100.0, string(Percentile95Latency): 200.0, + string(Percentile99Latency): 280.0, string(MaxLatency): 300.0, string(ErrorRate): 1.0, } @@ -1431,6 +1436,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 150.0, // 50% increase string(Percentile95Latency): 300.0, // 50% increase + string(Percentile99Latency): 420.0, // 50% increase string(MaxLatency): 450.0, // 50% increase string(ErrorRate): 2.0, // 100% increase } @@ -1440,11 +1446,12 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 50.0000%% different, which is higher than the threshold", string(MedianLatency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 50.0000%% different, which is higher than the threshold", string(Percentile95Latency))) + assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 50.0000%% different, which is higher than the threshold", string(Percentile99Latency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 50.0000%% different, which is higher than the threshold", string(MaxLatency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 100.0000%% different, which is higher than the threshold", string(ErrorRate))) }) @@ -1465,6 +1472,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 0.0, string(Percentile95Latency): 0.0, + string(Percentile99Latency): 0.0, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1489,6 +1497,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 0.0, string(Percentile95Latency): 0.0, + string(Percentile99Latency): 0.0, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1498,7 +1507,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) assert.False(t, failed) assert.Nil(t, err) }) @@ -1519,6 +1528,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 100.0, string(Percentile95Latency): 0.0, + string(Percentile99Latency): 0.0, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1550,10 +1560,11 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from current report", string(Percentile95Latency))) + assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from current report", string(Percentile99Latency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from current report", string(MaxLatency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from current report", string(ErrorRate))) }) @@ -1596,6 +1607,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 105.0, string(Percentile95Latency): 0.0, + string(Percentile99Latency): 0.0, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1605,10 +1617,11 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from previous report", string(Percentile95Latency))) + assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from previous report", string(Percentile99Latency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from previous report", string(MaxLatency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from previous report", string(ErrorRate))) }) @@ -1629,6 +1642,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 100.0, string(Percentile95Latency): 0.0, + string(Percentile99Latency): 0.0, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1660,10 +1674,11 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from current report", string(Percentile95Latency))) + assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from current report", string(Percentile99Latency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from current report", string(MaxLatency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s metric results were missing from current report", string(ErrorRate))) }) @@ -1684,6 +1699,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 0.0, string(Percentile95Latency): 0.0, + string(Percentile99Latency): 0.0, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1708,6 +1724,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 100.0, string(Percentile95Latency): 200.0, + string(Percentile99Latency): 280.0, string(MaxLatency): 300.0, string(ErrorRate): 1.0, } @@ -1717,11 +1734,12 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 999.0000%% different, which is higher than the threshold", string(MedianLatency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 999.0000%% different, which is higher than the threshold", string(Percentile95Latency))) + assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 999.0000%% different, which is higher than the threshold", string(Percentile99Latency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 999.0000%% different, which is higher than the threshold", string(MaxLatency))) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 999.0000%% different, which is higher than the threshold", string(ErrorRate))) }) @@ -1742,6 +1760,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 10.0, string(Percentile95Latency): 20.0, + string(Percentile99Latency): 28.0, string(MaxLatency): 311.0, string(ErrorRate): 1.0, } @@ -1766,6 +1785,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 0.0, string(Percentile95Latency): 0.0, + string(Percentile99Latency): 0.0, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1775,7 +1795,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) assert.False(t, failed) assert.Nil(t, err) }) @@ -1796,6 +1816,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 10.1, string(Percentile95Latency): 10.1, + string(Percentile99Latency): 10.1, string(MaxLatency): 10.0, string(ErrorRate): 10.0, } @@ -1820,6 +1841,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 10.2, string(Percentile95Latency): 10.1999, + string(Percentile99Latency): 10.19999, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1829,7 +1851,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(0.99, 0.9892, 10.0, 10.0, currentReport, previousReport) + failed, err := CompareDirectWithThresholds(0.99, 0.9892, 0.977, 10.0, 10.0, currentReport, previousReport) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("[test-gen] %s is 0.9901%% different, which is higher than the threshold", string(MedianLatency))) @@ -1851,6 +1873,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 10.2, string(Percentile95Latency): 10.1999, + string(Percentile99Latency): 280.0, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1860,17 +1883,17 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, report, nil) + failed, err := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, report, nil) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), "one or both reports are nil") - failed, err = CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, nil, report) + failed, err = CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, nil, report) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), "one or both reports are nil") - failed, err = CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, nil, nil) + failed, err = CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, nil, nil) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), "one or both reports are nil") @@ -1892,6 +1915,7 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { return map[string]interface{}{ string(MedianLatency): 10.0, string(Percentile95Latency): 10.0, + string(Percentile99Latency): 10.0, string(MaxLatency): 0.0, string(ErrorRate): 0.0, } @@ -1901,21 +1925,22 @@ func TestBenchSpy_CompareDirectWithThresholds(t *testing.T) { }, } - failed, err := CompareDirectWithThresholds(-0.1, 100.0, 0.0, 100.0, report, report) + failed, err := CompareDirectWithThresholds(-0.1, 100.0, 0.0, 100.0, 100.0, report, report) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), "median threshold -0.1000 is not in the range [0, 100]") - failed, err = CompareDirectWithThresholds(1.0, 101.0, 0.0, 100.0, report, report) + failed, err = CompareDirectWithThresholds(1.0, 101.0, 0.0, 100.0, 10.0, report, report) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), "p95 threshold 101.0000 is not in the range [0, 100]") - failed, err = CompareDirectWithThresholds(-1, -1, -1, -1, report, report) + failed, err = CompareDirectWithThresholds(-1, -1, -1, -1, -1, report, report) assert.True(t, failed) assert.NotNil(t, err) assert.Contains(t, err.Error(), "median threshold -1.0000 is not in the range [0, 100]") assert.Contains(t, err.Error(), "p95 threshold -1.0000 is not in the range [0, 100]") + assert.Contains(t, err.Error(), "p99 threshold -1.0000 is not in the range [0, 100]") assert.Contains(t, err.Error(), "max threshold -1.0000 is not in the range [0, 100]") assert.Contains(t, err.Error(), "error rate threshold -1.0000 is not in the range [0, 100]") }) @@ -2001,6 +2026,6 @@ func TestBenchSpy_Standard_Direct_Metrics_Two_Generators_E2E(t *testing.T) { fetchErr = currentReport.FetchData(fetchCtx) require.NoError(t, fetchErr, "failed to fetch data for original report") - hasErrors, errors := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + hasErrors, errors := CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) require.False(t, hasErrors, fmt.Sprintf("errors found: %v", errors)) } diff --git a/wasp/benchspy/types.go b/wasp/benchspy/types.go index e44d4579e..acc28c577 100644 --- a/wasp/benchspy/types.go +++ b/wasp/benchspy/types.go @@ -63,11 +63,12 @@ type StandardLoadMetric string const ( MedianLatency StandardLoadMetric = "median_latency" Percentile95Latency StandardLoadMetric = "95th_percentile_latency" + Percentile99Latency StandardLoadMetric = "99th_percentile_latency" MaxLatency StandardLoadMetric = "max_latency" ErrorRate StandardLoadMetric = "error_rate" ) -var StandardLoadMetrics = []StandardLoadMetric{MedianLatency, Percentile95Latency, MaxLatency, ErrorRate} +var StandardLoadMetrics = []StandardLoadMetric{MedianLatency, Percentile95Latency, Percentile99Latency, MaxLatency, ErrorRate} type StandardResourceMetric string diff --git a/wasp/examples/benchspy/direct_query_executor/direct_query_executor_test.go b/wasp/examples/benchspy/direct_query_executor/direct_query_executor_test.go index 9c8536e0f..481ed5939 100644 --- a/wasp/examples/benchspy/direct_query_executor/direct_query_executor_test.go +++ b/wasp/examples/benchspy/direct_query_executor/direct_query_executor_test.go @@ -62,6 +62,7 @@ func TestBenchSpy_Standard_Direct_Metrics(t *testing.T) { defer cancelFn() // currentReport is the report that we just created (baseLineReport) + // this will fail unless previous report has been committed currentReport, previousReport, err := benchspy.FetchNewStandardReportAndLoadLatestPrevious( fetchCtx, "v2", @@ -73,7 +74,7 @@ func TestBenchSpy_Standard_Direct_Metrics(t *testing.T) { // make sure that previous report is the same as the baseline report require.Equal(t, baseLineReport.CommitOrTag, previousReport.CommitOrTag, "current report should be the same as the original report") - hasErrors, errors := benchspy.CompareDirectWithThresholds(1.0, 1.0, 1.0, 1.0, currentReport, previousReport) + hasErrors, errors := benchspy.CompareDirectWithThresholds(1.0, 1.0, 1.0, 1.0, 1.0, currentReport, previousReport) require.False(t, hasErrors, fmt.Sprintf("errors found: %v", errors)) } @@ -163,6 +164,6 @@ func TestBenchSpy_Standard_Direct_Metrics_Two_Generators(t *testing.T) { // make sure that previous report is the same as the baseline report require.Equal(t, baseLineReport.CommitOrTag, previousReport.CommitOrTag, "current report should be the same as the original report") - hasErrors, errors := benchspy.CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, currentReport, previousReport) + hasErrors, errors := benchspy.CompareDirectWithThresholds(10.0, 10.0, 10.0, 10.0, 10.0, currentReport, previousReport) require.False(t, hasErrors, fmt.Sprintf("errors found: %v", errors)) } diff --git a/wasp/examples/benchspy/direct_query_executor/direct_query_real_case_test.go b/wasp/examples/benchspy/direct_query_executor/direct_query_real_case_test.go index 30d062e7c..f403a4037 100644 --- a/wasp/examples/benchspy/direct_query_executor/direct_query_real_case_test.go +++ b/wasp/examples/benchspy/direct_query_executor/direct_query_real_case_test.go @@ -69,7 +69,7 @@ package main // ) // require.NoError(t, err, "failed to fetch current report or load the previous one") // -// hasErrors, errors := benchspy.CompareDirectWithThresholds(1.0, 1.0, 1.0, 1.0, currentReport, previousReport) +// hasErrors, errors := benchspy.CompareDirectWithThresholds(1.0, 1.0, 1.0, 1.0, 1.0, currentReport, previousReport) // require.True(t, hasErrors, "Found no errors, but expected some") // require.Equal(t, 3, len(errors), "Expected 3 errors, got %d", len(errors)) // diff --git a/wasp/examples/benchspy/loki_query_executor/loki_query_executor_test.go b/wasp/examples/benchspy/loki_query_executor/loki_query_executor_test.go index 06331b6b4..e88a75208 100644 --- a/wasp/examples/benchspy/loki_query_executor/loki_query_executor_test.go +++ b/wasp/examples/benchspy/loki_query_executor/loki_query_executor_test.go @@ -96,6 +96,7 @@ func TestBenchSpy_Standard_Loki_Metrics(t *testing.T) { compareAverages(t, string(benchspy.MedianLatency), currentAsStringSlice, previousAsStringSlice, 1.0) compareAverages(t, string(benchspy.Percentile95Latency), currentAsStringSlice, previousAsStringSlice, 1.0) + compareAverages(t, string(benchspy.Percentile99Latency), currentAsStringSlice, previousAsStringSlice, 1.0) compareAverages(t, string(benchspy.MaxLatency), currentAsStringSlice, previousAsStringSlice, 1.0) compareAverages(t, string(benchspy.ErrorRate), currentAsStringSlice, previousAsStringSlice, 1.0) } @@ -331,6 +332,7 @@ func TestBenchSpy_Standard_Loki_Metrics_Two_Generators(t *testing.T) { compareAverages(t, string(benchspy.MedianLatency), currentAsStringSlice_vu1, previousAsStringSlice_vu1, 10.0) compareAverages(t, string(benchspy.Percentile95Latency), currentAsStringSlice_vu1, previousAsStringSlice_vu1, 10.0) + compareAverages(t, string(benchspy.Percentile99Latency), currentAsStringSlice_vu1, previousAsStringSlice_vu1, 10.0) compareAverages(t, string(benchspy.MaxLatency), currentAsStringSlice_vu1, previousAsStringSlice_vu1, 10.0) compareAverages(t, string(benchspy.ErrorRate), currentAsStringSlice_vu1, previousAsStringSlice_vu1, 10.0) @@ -339,6 +341,7 @@ func TestBenchSpy_Standard_Loki_Metrics_Two_Generators(t *testing.T) { compareAverages(t, string(benchspy.MedianLatency), currentAsStringSlice_vu2, previousAsStringSlice_vu2, 10.0) compareAverages(t, string(benchspy.Percentile95Latency), currentAsStringSlice_vu2, previousAsStringSlice_vu2, 10.0) + compareAverages(t, string(benchspy.Percentile99Latency), currentAsStringSlice_vu2, previousAsStringSlice_vu2, 10.0) compareAverages(t, string(benchspy.MaxLatency), currentAsStringSlice_vu2, previousAsStringSlice_vu2, 10.0) compareAverages(t, string(benchspy.ErrorRate), currentAsStringSlice_vu2, previousAsStringSlice_vu2, 10.0) }