feat(testing): add benchmark markers

Author: not-matthias

testing/capi/instrument-hooks.go

@@ -7,10 +7,18 @@ typedef struct instruments_root_InstrumentHooks__547 InstrumentHooks;
 */
 import "C"
 import (
+	"os"
 	"runtime"
 	"unsafe"
 )
 
+const (
+	MarkerTypeSampleStart    = 0
+	MarkerTypeSampleEnd      = 1
+	MarkerTypeBenchmarkStart = 2
+	MarkerTypeBenchmarkEnd   = 3
+)
+
 // This will be set in the go-runner
 var integrationVersion = "dev"
 
@@ -79,3 +87,16 @@ func (i *InstrumentHooks) IsInstrumented() bool {
 	}
 	return bool(C.instrument_hooks_is_instrumented(i.hooks))
 }
+
+func CurrentTimestamp() uint64 {
+	return uint64(C.instrument_hooks_current_timestamp())
+}
+
+func (i *InstrumentHooks) AddBenchmarkTimestamps(startTimestamp, endTimestamp uint64) {
+	if i.hooks == nil {
+		return
+	}
+	pid := uint32(os.Getpid())
+	C.instrument_hooks_add_marker(i.hooks, C.uint32_t(pid), C.uint8_t(MarkerTypeBenchmarkStart), C.uint64_t(startTimestamp))
+	C.instrument_hooks_add_marker(i.hooks, C.uint32_t(pid), C.uint8_t(MarkerTypeBenchmarkEnd), C.uint64_t(endTimestamp))
+}

testing/testing/benchmark.go

@@ -90,6 +90,10 @@ type codspeed struct {
 
 	codspeedTimePerRoundNs []time.Duration
 	codspeedItersPerRound  []int64
+
+	startTimestamp  uint64
+	startTimestamps []uint64
+	stopTimestamps  []uint64
 }
 
 // B is a type passed to [Benchmark] functions to manage benchmark
@@ -165,19 +169,20 @@ func (b *B) StartTimer() {
 // while performing steps that you don't want to measure.
 func (b *B) StopTimer() {
 	if b.timerOn {
-		// For b.N loops: This will be called in runN which sets b.N to the number of iterations.
-		// For b.Loop() loops: loopSlowPath sets b.N to 0 to prevent b.N loops within b.Loop. However, since
-		// we're starting/stopping the timer for each iteration in the b.Loop() loop, we can use 1 as
-		// the number of iterations for this round.
-		b.codspeedItersPerRound = append(b.codspeedItersPerRound, max(int64(b.N), 1))
-		b.codspeedTimePerRoundNs = append(b.codspeedTimePerRoundNs, highPrecisionTimeSince(b.start))
 		b.duration += highPrecisionTimeSince(b.start)
 		// runtime.ReadMemStats(&memStats)
 		// b.netAllocs += memStats.Mallocs - b.startAllocs
 		// b.netBytes += memStats.TotalAlloc - b.startBytes
 		b.timerOn = false
 		// If we hit B.Loop with the timer stopped, fail.
 		// b.loop.i |= loopPoisonTimer
+
+		// For b.N loops: This will be called in runN which sets b.N to the number of iterations.
+		// For b.Loop() loops: loopSlowPath sets b.N to 0 to prevent b.N loops within b.Loop. However, since
+		// we're starting/stopping the timer for each iteration in the b.Loop() loop, we can use 1 as
+		// the number of iterations for this round.
+		b.codspeedItersPerRound = append(b.codspeedItersPerRound, max(int64(b.N), 1))
+		b.codspeedTimePerRoundNs = append(b.codspeedTimePerRoundNs, highPrecisionTimeSince(b.start))
 	}
 }
 
@@ -197,10 +202,29 @@ func (b *B) ResetTimer() {
 		b.startAllocs = memStats.Mallocs
 		b.startBytes = memStats.TotalAlloc
 		b.start = highPrecisionTimeNow()
+
+		b.startTimestamp = capi.CurrentTimestamp()
 	}
 	b.duration = 0
 	b.netAllocs = 0
 	b.netBytes = 0
+
+	// Clear CodSpeed timestamp data
+	b.codspeedItersPerRound = b.codspeedItersPerRound[:0]
+	b.codspeedTimePerRoundNs = b.codspeedTimePerRoundNs[:0]
+	b.startTimestamps = b.startTimestamps[:0]
+	b.stopTimestamps = b.stopTimestamps[:0]
+}
+
+func (b *B) sendAccumulatedTimestamps() {
+	for i := 0; i < len(b.startTimestamps); i++ {
+		b.instrument_hooks.AddBenchmarkTimestamps(
+			b.startTimestamps[i],
+			b.stopTimestamps[i],
+		)
+	}
+	b.startTimestamps = b.startTimestamps[:0]
+	b.stopTimestamps = b.stopTimestamps[:0]
 }
 
 // SetBytes records the number of bytes processed in a single operation.
@@ -242,11 +266,23 @@ func (b *B) __codspeed_root_frame__runN(n int) {
 	b.parallelism = 1
 	b.ResetTimer()
 	b.StartTimer()
+	startTimestamp := capi.CurrentTimestamp()
 	b.benchFunc(b)
+	endTimestamp = capi.CurrentTimestamp()
 	b.StopTimer()
 	b.previousN = n
 	b.previousDuration = b.duration
 
+	if b.startTimestamp >= endTimestamp {
+		// This should never happen, unless we have a bug in the timer logic.
+		panic(fmt.Sprintf("Invalid benchmark timestamps: start timestamp (%d) is greater than or equal to end timestamp (%d)", b.startTimestamp, endTimestamp))
+	}
+	b.startTimestamps = append(b.startTimestamps, b.startTimestamp)
+	b.stopTimestamps = append(b.stopTimestamps, endTimestamp)
+
+	// Reset to prevent accidental reuse
+	b.startTimestamp = 0
+
 	if b.loop.n > 0 && !b.loop.done && !b.failed {
 		b.Error("benchmark function returned without B.Loop() == false (break or return in loop?)")
 	}
@@ -385,8 +421,7 @@ func (b *B) launch() {
 			}
 
 			// Reset the fields from the warmup run
-			b.codspeedItersPerRound = make([]int64, 0)
-			b.codspeedTimePerRoundNs = make([]time.Duration, 0)
+			b.ResetTimer()
 
 			// Final run:
 			benchD := b.benchTime.d
@@ -411,6 +446,7 @@ func (b *B) launch() {
 				b.runN(int(roundN))
 			}
 			b.codspeed.instrument_hooks.StopBenchmark()
+			b.sendAccumulatedTimestamps()
 		}
 	}
 	b.result = BenchmarkResult{b.N, b.duration, b.bytes, b.netAllocs, b.netBytes, b.codspeedTimePerRoundNs, b.codspeedItersPerRound, b.extra}
@@ -452,6 +488,8 @@ func (b *B) stopOrScaleBLoop() bool {
 		// Stop the timer so we don't count cleanup time
 		b.StopTimer()
 		b.codspeed.instrument_hooks.StopBenchmark()
+		b.sendAccumulatedTimestamps()
+
 		// Commit iteration count
 		b.N = int(b.loop.n)
 		b.loop.done = true
@@ -490,9 +528,6 @@ func (b *B) loopSlowPath() bool {
 		b.N = 0
 		b.loop.i++
 
-		b.codspeedItersPerRound = make([]int64, 0)
-		b.codspeedTimePerRoundNs = make([]time.Duration, 0)
-
 		b.codspeed.instrument_hooks.StartBenchmark()
 		b.ResetTimer()
 		b.StartTimer()
@@ -508,6 +543,8 @@ func (b *B) loopSlowPath() bool {
 		}
 		b.StopTimer()
 		b.codspeed.instrument_hooks.StopBenchmark()
+		b.sendAccumulatedTimestamps()
+
 		// Commit iteration count
 		b.N = int(b.loop.n)
 		b.loop.done = true