feat(testing): add benchmark markers

Author: not-matthias

testing/capi/instrument-hooks.go

@@ -7,10 +7,18 @@ typedef struct instruments_root_InstrumentHooks__547 InstrumentHooks;
 */
 import "C"
 import (
+	"os"
 	"runtime"
 	"unsafe"
 )
 
+const (
+	MarkerTypeSampleStart    = 0
+	MarkerTypeSampleEnd      = 1
+	MarkerTypeBenchmarkStart = 2
+	MarkerTypeBenchmarkEnd   = 3
+)
+
 // This will be set in the go-runner
 var integrationVersion = "dev"
 
@@ -79,3 +87,16 @@ func (i *InstrumentHooks) IsInstrumented() bool {
 	}
 	return bool(C.instrument_hooks_is_instrumented(i.hooks))
 }
+
+func CurrentTimestamp() uint64 {
+	return uint64(C.instrument_hooks_current_timestamp())
+}
+
+func (i *InstrumentHooks) AddBenchmarkTimestamps(startTimestamp, endTimestamp uint64) {
+	if i.hooks == nil {
+		return
+	}
+	pid := uint32(os.Getpid())
+	C.instrument_hooks_add_marker(i.hooks, C.uint32_t(pid), C.uint8_t(MarkerTypeBenchmarkStart), C.uint64_t(startTimestamp))
+	C.instrument_hooks_add_marker(i.hooks, C.uint32_t(pid), C.uint8_t(MarkerTypeBenchmarkEnd), C.uint64_t(endTimestamp))
+}

testing/testing/benchmark.go

@@ -90,6 +90,10 @@ type codspeed struct {
 
 	codspeedTimePerRoundNs []time.Duration
 	codspeedItersPerRound  []int64
+
+	startTimestamp  uint64
+	startTimestamps []uint64
+	stopTimestamps  []uint64
 }
 
 // B is a type passed to [Benchmark] functions to manage benchmark
@@ -158,26 +162,40 @@ func (b *B) StartTimer() {
 		b.start = highPrecisionTimeNow()
 		b.timerOn = true
 		// b.loop.i &^= loopPoisonTimer
+		b.startTimestamp = capi.CurrentTimestamp()
 	}
 }
 
 // StopTimer stops timing a test. This can be used to pause the timer
 // while performing steps that you don't want to measure.
 func (b *B) StopTimer() {
 	if b.timerOn {
-		// For b.N loops: This will be called in runN which sets b.N to the number of iterations.
-		// For b.Loop() loops: loopSlowPath sets b.N to 0 to prevent b.N loops within b.Loop. However, since
-		// we're starting/stopping the timer for each iteration in the b.Loop() loop, we can use 1 as
-		// the number of iterations for this round.
-		b.codspeedItersPerRound = append(b.codspeedItersPerRound, max(int64(b.N), 1))
-		b.codspeedTimePerRoundNs = append(b.codspeedTimePerRoundNs, highPrecisionTimeSince(b.start))
+		endTimestamp := capi.CurrentTimestamp()
 		b.duration += highPrecisionTimeSince(b.start)
 		// runtime.ReadMemStats(&memStats)
 		// b.netAllocs += memStats.Mallocs - b.startAllocs
 		// b.netBytes += memStats.TotalAlloc - b.startBytes
 		b.timerOn = false
 		// If we hit B.Loop with the timer stopped, fail.
 		// b.loop.i |= loopPoisonTimer
+
+		if b.startTimestamp >= endTimestamp {
+			// This should never happen, unless we have a bug in the
+			// timer logic.
+			panic("Invalid benchmark start timestamp")
+		}
+		b.startTimestamps = append(b.startTimestamps, b.startTimestamp)
+		b.stopTimestamps = append(b.stopTimestamps, endTimestamp)
+
+		// Reset to prevent accidental reuse
+		b.startTimestamp = 0
+
+		// For b.N loops: This will be called in runN which sets b.N to the number of iterations.
+		// For b.Loop() loops: loopSlowPath sets b.N to 0 to prevent b.N loops within b.Loop. However, since
+		// we're starting/stopping the timer for each iteration in the b.Loop() loop, we can use 1 as
+		// the number of iterations for this round.
+		b.codspeedItersPerRound = append(b.codspeedItersPerRound, max(int64(b.N), 1))
+		b.codspeedTimePerRoundNs = append(b.codspeedTimePerRoundNs, highPrecisionTimeSince(b.start))
 	}
 }
 
@@ -201,6 +219,23 @@ func (b *B) ResetTimer() {
 	b.duration = 0
 	b.netAllocs = 0
 	b.netBytes = 0
+
+	// Clear CodSpeed timestamp data
+	b.codspeedItersPerRound = b.codspeedItersPerRound[:0]
+	b.codspeedTimePerRoundNs = b.codspeedTimePerRoundNs[:0]
+	b.startTimestamps = b.startTimestamps[:0]
+	b.stopTimestamps = b.stopTimestamps[:0]
+}
+
+func (b *B) sendAccumulatedTimestamps() {
+	for i := 0; i < len(b.startTimestamps); i++ {
+		b.instrument_hooks.AddBenchmarkTimestamps(
+			b.startTimestamps[i],
+			b.stopTimestamps[i],
+		)
+	}
+	b.startTimestamps = b.startTimestamps[:0]
+	b.stopTimestamps = b.stopTimestamps[:0]
 }
 
 // SetBytes records the number of bytes processed in a single operation.
@@ -385,8 +420,7 @@ func (b *B) launch() {
 			}
 
 			// Reset the fields from the warmup run
-			b.codspeedItersPerRound = make([]int64, 0)
-			b.codspeedTimePerRoundNs = make([]time.Duration, 0)
+			b.ResetTimer()
 
 			// Final run:
 			benchD := b.benchTime.d
@@ -411,6 +445,7 @@ func (b *B) launch() {
 				b.runN(int(roundN))
 			}
 			b.codspeed.instrument_hooks.StopBenchmark()
+			b.sendAccumulatedTimestamps()
 		}
 	}
 	b.result = BenchmarkResult{b.N, b.duration, b.bytes, b.netAllocs, b.netBytes, b.codspeedTimePerRoundNs, b.codspeedItersPerRound, b.extra}
@@ -452,6 +487,8 @@ func (b *B) stopOrScaleBLoop() bool {
 		// Stop the timer so we don't count cleanup time
 		b.StopTimer()
 		b.codspeed.instrument_hooks.StopBenchmark()
+		b.sendAccumulatedTimestamps()
+
 		// Commit iteration count
 		b.N = int(b.loop.n)
 		b.loop.done = true
@@ -490,9 +527,7 @@ func (b *B) loopSlowPath() bool {
 		b.N = 0
 		b.loop.i++
 
-		b.codspeedItersPerRound = make([]int64, 0)
-		b.codspeedTimePerRoundNs = make([]time.Duration, 0)
-
+		b.ResetTimer()
 		b.codspeed.instrument_hooks.StartBenchmark()
 		b.ResetTimer()
 		b.StartTimer()
@@ -509,6 +544,8 @@ func (b *B) loopSlowPath() bool {
 		}
 		b.StopTimer()
 		b.codspeed.instrument_hooks.StopBenchmark()
+		b.sendAccumulatedTimestamps()
+
 		// Commit iteration count
 		b.N = int(b.loop.n)
 		b.loop.done = true