[dyninst] Expose more dyninst debug data via flares

pkg/dyninst/actuator/actuator.go

@@ -99,6 +99,22 @@ func (a *Actuator) Stats() map[string]any {
 	}
 }
 
+// DebugInfo returns a snapshot of the actuator's internal state for debugging.
+func (a *Actuator) DebugInfo() *DebugInfo {
+	debugInfoChan := make(chan DebugInfo, 1)
+	select {
+	case <-a.shuttingDown:
+		return nil
+	case a.events <- eventGetDebugInfo{debugInfoChan: debugInfoChan}:
+		select {
+		case <-a.shuttingDown:
+			return nil
+		case info := <-debugInfoChan:
+			return &info
+		}
+	}
+}
+
 // NewActuator creates a new Actuator instance.
 func NewActuator(cfg Config) *Actuator {
 	if cfg.DiscoveredTypesLimit == 0 {

pkg/dyninst/actuator/debug_info.go

@@ -0,0 +1,129 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+//go:build linux_bpf
+
+package actuator
+
+import (
+	"slices"
+
+	"github.com/DataDog/datadog-agent/pkg/dyninst/ir"
+)
+
+// DebugInfo contains a snapshot of the actuator's internal state for
+// debugging purposes, exposed via the flare mechanism.
+type DebugInfo struct {
+	Processes        []ProcessDebugInfo  `json:"processes"`
+	Programs         []ProgramDebugInfo  `json:"programs"`
+	DiscoveredTypes  map[string][]string `json:"discovered_types"`
+	CurrentlyLoading *ir.ProgramID       `json:"currently_loading"`
+	QueuedLoading    []ir.ProgramID      `json:"queued_loading"`
+	Counters         CountersDebugInfo   `json:"counters"`
+	CircuitBreaker   CircuitBreakerInfo  `json:"circuit_breaker"`
+}
+
+// ProcessDebugInfo contains debug information about a single process in the
+// actuator's state machine.
+type ProcessDebugInfo struct {
+	PID            int32        `json:"pid"`
+	State          string       `json:"state"`
+	Service        string       `json:"service"`
+	CurrentProgram ir.ProgramID `json:"current_program"`
+	ProbeCount     int          `json:"probe_count"`
+}
+
+// ProgramDebugInfo contains debug information about a single program in the
+// actuator's state machine.
+type ProgramDebugInfo struct {
+	ProgramID          ir.ProgramID `json:"program_id"`
+	State              string       `json:"state"`
+	ProcessPID         int32        `json:"process_pid"`
+	ProbeCount         int          `json:"probe_count"`
+	NeedsRecompilation bool         `json:"needs_recompilation"`
+}
+
+// CountersDebugInfo contains cumulative counters from the actuator.
+type CountersDebugInfo struct {
+	Loaded                      uint64 `json:"loaded"`
+	LoadFailed                  uint64 `json:"load_failed"`
+	Attached                    uint64 `json:"attached"`
+	AttachFailed                uint64 `json:"attach_failed"`
+	Detached                    uint64 `json:"detached"`
+	Unloaded                    uint64 `json:"unloaded"`
+	TypeRecompilationsTriggered uint64 `json:"type_recompilations_triggered"`
+}
+
+// CircuitBreakerInfo contains the circuit breaker configuration.
+type CircuitBreakerInfo struct {
+	Interval          string  `json:"interval"`
+	PerProbeCPULimit  float64 `json:"per_probe_cpu_limit"`
+	AllProbesCPULimit float64 `json:"all_probes_cpu_limit"`
+	InterruptOverhead string  `json:"interrupt_overhead"`
+}
+
+// debugInfo returns a snapshot of the state machine for debugging.
+func (s *state) debugInfo() DebugInfo {
+	processes := make([]ProcessDebugInfo, 0, len(s.processes))
+	for _, p := range s.processes {
+		processes = append(processes, ProcessDebugInfo{
+			PID:            p.processID.PID,
+			State:          p.state.String(),
+			Service:        p.service,
+			CurrentProgram: p.currentProgram,
+			ProbeCount:     len(p.probes),
+		})
+	}
+
+	programs := make([]ProgramDebugInfo, 0, len(s.programs))
+	for _, p := range s.programs {
+		programs = append(programs, ProgramDebugInfo{
+			ProgramID:          p.id,
+			State:              p.state.String(),
+			ProcessPID:         p.processID.PID,
+			ProbeCount:         len(p.config),
+			NeedsRecompilation: p.needsRecompilation,
+		})
+	}
+
+	discoveredTypes := make(map[string][]string, len(s.discoveredTypes))
+	for svc, types := range s.discoveredTypes {
+		discoveredTypes[svc] = slices.Clone(types)
+	}
+
+	var currentlyLoading *ir.ProgramID
+	if s.currentlyLoading != nil {
+		id := s.currentlyLoading.id
+		currentlyLoading = &id
+	}
+
+	queuedLoading := make([]ir.ProgramID, 0, s.queuedLoading.len())
+	for _, item := range s.queuedLoading.m {
+		queuedLoading = append(queuedLoading, item.value.id)
+	}
+
+	return DebugInfo{
+		Processes:        processes,
+		Programs:         programs,
+		DiscoveredTypes:  discoveredTypes,
+		CurrentlyLoading: currentlyLoading,
+		QueuedLoading:    queuedLoading,
+		Counters: CountersDebugInfo{
+			Loaded:                      s.counters.loaded,
+			LoadFailed:                  s.counters.loadFailed,
+			Attached:                    s.counters.attached,
+			AttachFailed:                s.counters.attachFailed,
+			Detached:                    s.counters.detached,
+			Unloaded:                    s.counters.unloaded,
+			TypeRecompilationsTriggered: s.counters.typeRecompilationsTriggered,
+		},
+		CircuitBreaker: CircuitBreakerInfo{
+			Interval:          s.breakerCfg.Interval.String(),
+			PerProbeCPULimit:  s.breakerCfg.PerProbeCPULimit,
+			AllProbesCPULimit: s.breakerCfg.AllProbesCPULimit,
+			InterruptOverhead: s.breakerCfg.InterruptOverhead.String(),
+		},
+	}
+}

pkg/dyninst/actuator/debug_info_test.go

@@ -0,0 +1,126 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+//go:build linux_bpf
+
+package actuator
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/DataDog/datadog-agent/pkg/dyninst/ir"
+	procinfo "github.com/DataDog/datadog-agent/pkg/dyninst/process"
+	"github.com/DataDog/datadog-agent/pkg/dyninst/rcjson"
+)
+
+func TestStateDebugInfoEmpty(t *testing.T) {
+	s := newState(Config{DiscoveredTypesLimit: 512})
+	info := s.debugInfo()
+
+	assert.Empty(t, info.Processes)
+	assert.Empty(t, info.Programs)
+	assert.Empty(t, info.DiscoveredTypes)
+	assert.Nil(t, info.CurrentlyLoading)
+	assert.Empty(t, info.QueuedLoading)
+	assert.Equal(t, uint64(0), info.Counters.Loaded)
+}
+
+func TestStateDebugInfoWithProcessesAndPrograms(t *testing.T) {
+	s := newState(Config{DiscoveredTypesLimit: 512})
+
+	pid := procinfo.ID{PID: 42}
+	probe := &rcjson.SnapshotProbe{
+		LogProbeCommon: rcjson.LogProbeCommon{
+			ProbeCommon: rcjson.ProbeCommon{
+				ID:      "probe-1",
+				Version: 1,
+				Where:   &rcjson.Where{MethodName: "main"},
+			},
+		},
+	}
+	s.processes[pid] = &process{
+		processID: pid,
+		state:     processStateAttached,
+		service:   "my-service",
+		probes:    map[probeKey]ir.ProbeDefinition{{id: "probe-1", version: 1}: probe},
+	}
+
+	progID := ir.ProgramID(1)
+	s.programs[progID] = &program{
+		id:        progID,
+		state:     programStateLoaded,
+		processID: pid,
+		config:    []ir.ProbeDefinition{probe},
+	}
+	s.processes[pid].currentProgram = progID
+
+	s.discoveredTypes["my-service"] = []string{"MyType", "OtherType"}
+
+	info := s.debugInfo()
+
+	assert.Len(t, info.Processes, 1)
+	assert.Equal(t, int32(42), info.Processes[0].PID)
+	assert.Equal(t, "Attached", info.Processes[0].State)
+	assert.Equal(t, "my-service", info.Processes[0].Service)
+	assert.Equal(t, 1, info.Processes[0].ProbeCount)
+	assert.Equal(t, progID, info.Processes[0].CurrentProgram)
+
+	assert.Len(t, info.Programs, 1)
+	assert.Equal(t, progID, info.Programs[0].ProgramID)
+	assert.Equal(t, "Loaded", info.Programs[0].State)
+	assert.Equal(t, int32(42), info.Programs[0].ProcessPID)
+	assert.Equal(t, 1, info.Programs[0].ProbeCount)
+	assert.False(t, info.Programs[0].NeedsRecompilation)
+
+	assert.Equal(t, []string{"MyType", "OtherType"}, info.DiscoveredTypes["my-service"])
+	assert.Nil(t, info.CurrentlyLoading)
+	assert.Empty(t, info.QueuedLoading)
+}
+
+func TestStateDebugInfoCounters(t *testing.T) {
+	s := newState(Config{DiscoveredTypesLimit: 512})
+	s.counters.loaded = 5
+	s.counters.loadFailed = 2
+	s.counters.attached = 3
+	s.counters.typeRecompilationsTriggered = 1
+
+	info := s.debugInfo()
+
+	assert.Equal(t, uint64(5), info.Counters.Loaded)
+	assert.Equal(t, uint64(2), info.Counters.LoadFailed)
+	assert.Equal(t, uint64(3), info.Counters.Attached)
+	assert.Equal(t, uint64(1), info.Counters.TypeRecompilationsTriggered)
+}
+
+func TestStateDebugInfoQueuedPrograms(t *testing.T) {
+	s := newState(Config{DiscoveredTypesLimit: 512})
+
+	pid := procinfo.ID{PID: 10}
+	prog := &program{
+		id:        ir.ProgramID(7),
+		state:     programStateQueued,
+		processID: pid,
+	}
+	s.programs[prog.id] = prog
+	s.queuedLoading.pushBack(prog)
+
+	info := s.debugInfo()
+
+	assert.Len(t, info.QueuedLoading, 1)
+	assert.Equal(t, ir.ProgramID(7), info.QueuedLoading[0])
+	assert.Nil(t, info.CurrentlyLoading)
+}
+
+func TestStateDebugInfoCurrentlyLoading(t *testing.T) {
+	s := newState(Config{DiscoveredTypesLimit: 512})
+	s.currentlyLoading = &program{id: ir.ProgramID(3)}
+
+	info := s.debugInfo()
+
+	assert.NotNil(t, info.CurrentlyLoading)
+	assert.Equal(t, ir.ProgramID(3), *info.CurrentlyLoading)
+}

pkg/dyninst/actuator/state.go

@@ -382,6 +382,10 @@ func handleEvent(
 		ev.metricsChan <- sm.Metrics()
 		return nil
 
+	case eventGetDebugInfo:
+		ev.debugInfoChan <- sm.debugInfo()
+		return nil
+
 	case eventHeartbeatCheck:
 		handleHeartbeatCheck(sm, effects)
 

pkg/dyninst/actuator/state_events.go

@@ -136,6 +136,15 @@ func (e eventGetMetrics) String() string {
 	return "eventGetMetrics{}"
 }
 
+type eventGetDebugInfo struct {
+	baseEvent
+	debugInfoChan chan<- DebugInfo
+}
+
+func (e eventGetDebugInfo) String() string {
+	return "eventGetDebugInfo{}"
+}
+
 type eventHeartbeatCheck struct {
 	baseEvent
 }