fix(testing): emit StopBenchmark command even when benches failed

Author: not-matthias

testing/fork.sh

@@ -120,6 +120,8 @@ find . -type f -name "*.go" -not -name "*_test.go" -exec sed -i 's|"testing"|tes
 # Restore CodSpeed-specific files
 restore_files "${CODSPEED_FILES[@]}"
 
+apply_patch "patches/benchmark_stopbenchmark_fail.patch" 10 ".."
+
 # Run pre-commit and format the code
 go fmt ./... > /dev/null 2>&1 || true
 pre-commit run --all-files > /dev/null 2>&1 || true

testing/patches/benchmark_stopbenchmark_fail.patch

@@ -0,0 +1,22 @@
+diff --git a/testing/testing/benchmark.go b/testing/testing/benchmark.go
+index e7e4d06..8fdea90 100644
+--- a/testing/testing/benchmark.go
++++ b/testing/testing/benchmark.go
+@@ -325,6 +325,8 @@ func (b *B) run1() bool {
+ 	}()
+ 	<-b.signal
+ 	if b.failed {
++		// This case can happen with a `b.Loop()` benchmark if any of the iterations fail
++		ensureBenchmarkIsStopped(b)
+ 		fmt.Fprintf(b.w, "%s--- FAIL: %s\n%s", b.chatty.prefix(), b.name, b.output)
+ 		return false
+ 	}
+@@ -888,6 +890,8 @@ func (s *benchState) processBench(b *B) {
+ 			}
+ 			r := b.doBench()
+ 			if b.failed {
++				ensureBenchmarkIsStopped(b)
++
+ 				// The output could be very long here, but probably isn't.
+ 				// We print it all, regardless, because we don't want to trim the reason
+ 				// the benchmark failed.

testing/testing/benchmark.go

@@ -325,6 +325,8 @@ func (b *B) run1() bool {
 	}()
 	<-b.signal
 	if b.failed {
+		// This case can happen with a `b.Loop()` benchmark if any of the iterations fail
+		ensureBenchmarkIsStopped(b)
 		fmt.Fprintf(b.w, "%s--- FAIL: %s\n%s", b.chatty.prefix(), b.name, b.output)
 		return false
 	}
@@ -888,6 +890,8 @@ func (s *benchState) processBench(b *B) {
 			}
 			r := b.doBench()
 			if b.failed {
+				ensureBenchmarkIsStopped(b)
+
 				// The output could be very long here, but probably isn't.
 				// We print it all, regardless, because we don't want to trim the reason
 				// the benchmark failed.

testing/testing/codspeed.go

@@ -66,3 +66,11 @@ func getGitRelativePath(absPath string) string {
 
 	return filepath.Join(goRunnerMetadata.RelativePackagePath, cwdRelativePath)
 }
+
+// If the benchmark execution failed, we have to ensure to stop the benchmark, which
+// will send the event to the runner to also stop perf. Otherwise we could possibly
+// sample a lot of data that isn't relevant. Additionally, we want to ensure that
+// the emitted markers are correct (otherwise we'd have a SampleStart without a SampleStop).
+func ensureBenchmarkIsStopped(b *B) {
+	b.codspeed.instrument_hooks.StopBenchmark()
+}