|
| 1 | +# BenchSpy - Standard Loki metrics |
| 2 | + |
| 3 | +> [!NOTE] |
| 4 | +> This example assumes you have access to Loki and Grafana instances. If you don't |
| 5 | +> find out how to launch them using CTFv2's [observability stack](../../../framework/observability/observability_stack.md). |
| 6 | +
|
| 7 | +Our Loki example, will vary from the previous one in just a couple of details: |
| 8 | +* generator will have Loki config |
| 9 | +* standard query executor type will be `benchspy.StandardQueryExecutor_Loki` |
| 10 | +* we will cast all results to `[]string` |
| 11 | +* and calculate medians for all metrics |
| 12 | + |
| 13 | +Ready? |
| 14 | + |
| 15 | +Let's define new load generation first: |
| 16 | +```go |
| 17 | +label := "benchspy-std" |
| 18 | + |
| 19 | +gen, err := wasp.NewGenerator(&wasp.Config{ |
| 20 | + T: t, |
| 21 | + // read Loki config from environment |
| 22 | + LokiConfig: wasp.NewEnvLokiConfig(), |
| 23 | + GenName: "vu", |
| 24 | + // set unique labels |
| 25 | + Labels: map[string]string{ |
| 26 | + "branch": label, |
| 27 | + "commit": label, |
| 28 | + }, |
| 29 | + CallTimeout: 100 * time.Millisecond, |
| 30 | + LoadType: wasp.VU, |
| 31 | + Schedule: wasp.Plain(10, 15*time.Second), |
| 32 | + VU: wasp.NewMockVU(&wasp.MockVirtualUserConfig{ |
| 33 | + CallSleep: 50 * time.Millisecond, |
| 34 | + }), |
| 35 | +}) |
| 36 | +require.NoError(t, err) |
| 37 | +``` |
| 38 | + |
| 39 | +Now let's run the generator and save baseline report: |
| 40 | +```go |
| 41 | +gen.Run(true) |
| 42 | + |
| 43 | +fetchCtx, cancelFn := context.WithTimeout(context.Background(), 60*time.Second) |
| 44 | +defer cancelFn() |
| 45 | + |
| 46 | +baseLineReport, err := benchspy.NewStandardReport( |
| 47 | + "c2cf545d733eef8bad51d685fcb302e277d7ca14", |
| 48 | + // notice the different standard executor type |
| 49 | + benchspy.WithStandardQueryExecutorType(benchspy.StandardQueryExecutor_Loki), |
| 50 | + benchspy.WithGenerators(gen), |
| 51 | +) |
| 52 | +require.NoError(t, err, "failed to create original report") |
| 53 | + |
| 54 | +fetchErr := baseLineReport.FetchData(fetchCtx) |
| 55 | +require.NoError(t, fetchErr, "failed to fetch data for original report") |
| 56 | + |
| 57 | +path, storeErr := baseLineReport.Store() |
| 58 | +require.NoError(t, storeErr, "failed to store current report", path) |
| 59 | +``` |
| 60 | + |
| 61 | +Since next steps are very similar to the ones used in the first test we will skip them and jump straight |
| 62 | +to metrics comparison. |
| 63 | + |
| 64 | +By default, `LokiQueryExecutor` returns `[]string` data type, so let's use dedicated convenience functions |
| 65 | +to cast them from `interface{}` to string slice: |
| 66 | +```go |
| 67 | +currentAsStringSlice := benchspy.MustAllLokiResults(currentReport) |
| 68 | +previousAsStringSlice := benchspy.MustAllLokiResults(previousReport) |
| 69 | +``` |
| 70 | + |
| 71 | +And finally, time to compare metrics. Since we have a `[]string` we will first convert it to `[]float64` and |
| 72 | +then calculate a median and assume it hasn't changed by more than 1%. Again, remember that this is just an illustration. |
| 73 | +You should decide yourself what's the best way to assert the metrics. |
| 74 | + |
| 75 | +```go |
| 76 | +var compareMedian = func(metricName string) { |
| 77 | + require.NotEmpty(t, currentAsStringSlice[metricName], "%s results were missing from current report", metricName) |
| 78 | + require.NotEmpty(t, previousAsStringSlice[metricName], "%s results were missing from previous report", metricName) |
| 79 | + |
| 80 | + currentFloatSlice, err := benchspy.StringSliceToFloat64Slice(currentAsStringSlice[metricName]) |
| 81 | + require.NoError(t, err, "failed to convert %s results to float64 slice", metricName) |
| 82 | + currentMedian := benchspy.CalculatePercentile(currentFloatSlice, 0.5) |
| 83 | + |
| 84 | + previousFloatSlice, err := benchspy.StringSliceToFloat64Slice(previousAsStringSlice[metricName]) |
| 85 | + require.NoError(t, err, "failed to convert %s results to float64 slice", metricName) |
| 86 | + previousMedian := benchspy.CalculatePercentile(previousFloatSlice, 0.5) |
| 87 | + |
| 88 | + var diffPrecentage float64 |
| 89 | + if previousMedian != 0 { |
| 90 | + diffPrecentage = (currentMedian - previousMedian) / previousMedian * 100 |
| 91 | + } else { |
| 92 | + diffPrecentage = currentMedian * 100 |
| 93 | + } |
| 94 | + assert.LessOrEqual(t, math.Abs(diffPrecentage), 1.0, "%s medians are more than 1% different", metricName, fmt.Sprintf("%.4f", diffPrecentage)) |
| 95 | +} |
| 96 | + |
| 97 | +compareMedian(string(benchspy.MedianLatency)) |
| 98 | +compareMedian(string(benchspy.Percentile95Latency)) |
| 99 | +compareMedian(string(benchspy.ErrorRate)) |
| 100 | +``` |
| 101 | + |
| 102 | +We have used standard metrics, which are the same as in the first test, now let's see how you can use your custom LogQl queries. |
| 103 | + |
| 104 | +You can find the full example [here](...). |
0 commit comments