Skip to content

Commit 21a2ce4

Browse files
herkolateganherkolategan
committed
microbench-ci: only compare the last run
Previously, if a regression occurred, the comparison step would compare all the runs instead of just the last one. The assumption was that if all runs show a regression we expect the combined result to show a regression, but since the baseline case shift with each run there's a small chance this does not always hold true. Hence, we revert here to only comparing the last run. Epic: None Release note: None
1 parent 6996186 commit 21a2ce4

File tree

2 files changed

+20
-178
lines changed

2 files changed

+20
-178
lines changed

‎pkg/cmd/microbench-ci/compare.go‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -86,18 +86,6 @@ func (c *CompareResult) status(metricName string) Status {
8686
return status
8787
}
8888

89-
// top returns the top status of all metrics in the comparison.
90-
func (c *CompareResult) top() Status {
91-
topStatus := NoChange
92-
for metric := range c.MetricMap {
93-
status := c.status(metric)
94-
if status > topStatus {
95-
topStatus = status
96-
}
97-
}
98-
return topStatus
99-
}
100-
10189
// compare compares the metrics of a benchmark between two revisions. Only the
10290
// specified last number of lines of the benchmark logs are considered. If lines
10391
// is 0, it considers the entire logs.
@@ -153,12 +141,6 @@ func (b Benchmarks) compareBenchmarks() (CompareResults, error) {
153141
if err != nil {
154142
return nil, err
155143
}
156-
if compareResult.top() != NoChange {
157-
compareResult, err = benchmark.compare(0)
158-
if err != nil {
159-
return nil, err
160-
}
161-
}
162144
compareResults = append(compareResults, compareResult)
163145
}
164146
return compareResults, nil

‎pkg/cmd/microbench-ci/testdata/regression.txt‎

Lines changed: 20 additions & 160 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,8 @@ run group=1
9494

9595
| Metric | Old Commit | New Commit | Delta | Note |
9696
|-----------------------------|----------------|----------------|------------|--------------|
97-
| 🔴 **sec/op** | 9.862m ±0% | 9.952m ±0% | +0.91% | p=0.000 n=30 |
98-
| ⚪ **allocs/op** | 10.38k ±0% | 10.38k ±0% | ~ | p=1.000 n=30 |
97+
| 🔴 **sec/op** | 9.862m ±1% | 9.952m ±0% | +0.91% | p=0.001 n=10 |
98+
| ⚪ **allocs/op** | 10.38k ±1% | 10.38k ±1% | ~ | p=1.000 n=10 |
9999

100100
<details><summary>Reproduce</summary>
101101

@@ -182,42 +182,22 @@ json
182182
"Metric": "B/op",
183183
"Summary": {
184184
"Center": 2367667,
185-
"Lo": 2365463,
186-
"Hi": 2368213,
187-
"Confidence": 0.95723,
185+
"Lo": 2358650,
186+
"Hi": 2370670,
187+
"Confidence": 0.97852,
188188
"Warnings": null
189189
},
190190
"Sample": {
191191
"Values": [
192192
2352326,
193-
2352326,
194-
2352326,
195-
2358650,
196193
2358650,
197-
2358650,
198-
2364281,
199194
2364281,
200-
2364281,
201-
2365463,
202195
2365463,
203-
2365463,
204-
2367582,
205196
2367582,
206-
2367582,
207-
2367752,
208197
2367752,
209-
2367752,
210-
2368213,
211198
2368213,
212-
2368213,
213-
2369187,
214199
2369187,
215-
2369187,
216-
2370670,
217200
2370670,
218-
2370670,
219-
2375306,
220-
2375306,
221201
2375306
222202
],
223203
"Thresholds": {
@@ -230,42 +210,22 @@ json
230210
"Metric": "allocs/op",
231211
"Summary": {
232212
"Center": 10378.50000,
233-
"Lo": 10377,
234-
"Hi": 10386,
235-
"Confidence": 0.95723,
213+
"Lo": 10287,
214+
"Hi": 10398,
215+
"Confidence": 0.97852,
236216
"Warnings": null
237217
},
238218
"Sample": {
239219
"Values": [
240220
10246,
241-
10246,
242-
10246,
243-
10287,
244221
10287,
245-
10287,
246-
10361,
247222
10361,
248-
10361,
249-
10377,
250223
10377,
251-
10377,
252-
10378,
253224
10378,
254-
10378,
255-
10379,
256225
10379,
257-
10379,
258-
10386,
259226
10386,
260-
10386,
261-
10392,
262227
10392,
263-
10392,
264-
10398,
265228
10398,
266-
10398,
267-
10411,
268-
10411,
269229
10411
270230
],
271231
"Thresholds": {
@@ -278,42 +238,22 @@ json
278238
"Metric": "sec/op",
279239
"Summary": {
280240
"Center": 0.00995,
281-
"Lo": 0.00995,
282-
"Hi": 0.00997,
283-
"Confidence": 0.95723,
241+
"Lo": 0.00993,
242+
"Hi": 0.00998,
243+
"Confidence": 0.97852,
284244
"Warnings": null
285245
},
286246
"Sample": {
287247
"Values": [
288248
0.00991,
289-
0.00991,
290-
0.00991,
291-
0.00993,
292249
0.00993,
293-
0.00993,
294-
0.00995,
295250
0.00995,
296251
0.00995,
297252
0.00995,
298253
0.00995,
299-
0.00995,
300-
0.00995,
301-
0.00995,
302-
0.00995,
303-
0.00995,
304-
0.00995,
305-
0.00995,
306-
0.00997,
307254
0.00997,
308-
0.00997,
309-
0.00998,
310-
0.00998,
311-
0.00998,
312255
0.00998,
313256
0.00998,
314-
0.00998,
315-
0.01000,
316-
0.01000,
317257
0.01000
318258
],
319259
"Thresholds": {
@@ -328,42 +268,22 @@ json
328268
"Metric": "B/op",
329269
"Summary": {
330270
"Center": 2367667,
331-
"Lo": 2365463,
332-
"Hi": 2368213,
333-
"Confidence": 0.95723,
271+
"Lo": 2358650,
272+
"Hi": 2370670,
273+
"Confidence": 0.97852,
334274
"Warnings": null
335275
},
336276
"Sample": {
337277
"Values": [
338278
2352326,
339-
2352326,
340-
2352326,
341-
2358650,
342279
2358650,
343-
2358650,
344-
2364281,
345280
2364281,
346-
2364281,
347-
2365463,
348281
2365463,
349-
2365463,
350-
2367582,
351282
2367582,
352-
2367582,
353-
2367752,
354283
2367752,
355-
2367752,
356-
2368213,
357-
2368213,
358284
2368213,
359285
2369187,
360-
2369187,
361-
2369187,
362286
2370670,
363-
2370670,
364-
2370670,
365-
2375306,
366-
2375306,
367287
2375306
368288
],
369289
"Thresholds": {
@@ -376,42 +296,22 @@ json
376296
"Metric": "allocs/op",
377297
"Summary": {
378298
"Center": 10378.50000,
379-
"Lo": 10377,
380-
"Hi": 10386,
381-
"Confidence": 0.95723,
299+
"Lo": 10287,
300+
"Hi": 10398,
301+
"Confidence": 0.97852,
382302
"Warnings": null
383303
},
384304
"Sample": {
385305
"Values": [
386-
10246,
387-
10246,
388306
10246,
389307
10287,
390-
10287,
391-
10287,
392-
10361,
393-
10361,
394308
10361,
395309
10377,
396-
10377,
397-
10377,
398-
10378,
399-
10378,
400310
10378,
401311
10379,
402-
10379,
403-
10379,
404-
10386,
405-
10386,
406312
10386,
407313
10392,
408-
10392,
409-
10392,
410-
10398,
411-
10398,
412314
10398,
413-
10411,
414-
10411,
415315
10411
416316
],
417317
"Thresholds": {
@@ -426,31 +326,11 @@ json
426326
"Center": 0,
427327
"Lo": 0,
428328
"Hi": 0,
429-
"Confidence": 0.95723,
329+
"Confidence": 0.97852,
430330
"Warnings": null
431331
},
432332
"Sample": {
433333
"Values": [
434-
0,
435-
0,
436-
0,
437-
0,
438-
0,
439-
0,
440-
0,
441-
0,
442-
0,
443-
0,
444-
0,
445-
0,
446-
0,
447-
0,
448-
0,
449-
0,
450-
0,
451-
0,
452-
0,
453-
0,
454334
0,
455335
0,
456336
0,
@@ -473,41 +353,21 @@ json
473353
"Summary": {
474354
"Center": 0.00986,
475355
"Lo": 0.00985,
476-
"Hi": 0.00988,
477-
"Confidence": 0.95723,
356+
"Hi": 0.00993,
357+
"Confidence": 0.97852,
478358
"Warnings": null
479359
},
480360
"Sample": {
481361
"Values": [
482362
0.00981,
483-
0.00981,
484-
0.00981,
485-
0.00985,
486-
0.00985,
487363
0.00985,
488364
0.00985,
489365
0.00985,
490366
0.00985,
491-
0.00985,
492-
0.00985,
493-
0.00985,
494-
0.00985,
495-
0.00985,
496-
0.00985,
497-
0.00987,
498-
0.00987,
499367
0.00987,
500368
0.00988,
501-
0.00988,
502-
0.00988,
503-
0.00990,
504-
0.00990,
505369
0.00990,
506370
0.00993,
507-
0.00993,
508-
0.00993,
509-
0.00998,
510-
0.00998,
511371
0.00998
512372
],
513373
"Thresholds": {

0 commit comments

Comments
 (0)