Generic Probe Support: Add kprobe Functionality Alongside exiting uprobes

cli_flags.go

@@ -71,7 +71,7 @@ var (
 	envVarsHelp = "Comma separated list of environment variables that will be reported with the" +
 		"captured profiling samples."
 	probeLinkHelper = "Attach a probe to a symbol of an executable. " +
-		"Expected format: /path/to/executable:symbol"
+		"Expected format: probe_type:target[:symbol]. probe_type can be kprobe, kretprobe, uprobe, or uretprobe."
 	loadProbeHelper = "Load generic eBPF program that can be attached externally to " +
 		"various user or kernel space hooks."
 )
@@ -133,8 +133,8 @@ func parseArgs() (*controller.Config, error) {
 
 	fs.StringVar(&args.IncludeEnvVars, "env-vars", defaultEnvVarsValue, envVarsHelp)
 
-	fs.Func("uprobe-link", probeLinkHelper, func(link string) error {
-		args.UProbeLinks = append(args.UProbeLinks, link)
+	fs.Func("probe-link", probeLinkHelper, func(link string) error {
+		args.ProbeLinks = append(args.ProbeLinks, link)
 		return nil
 	})
 

collector/config/config.go

@@ -19,7 +19,7 @@ type Config struct {
 	VerboseMode            bool          `mapstructure:"verbose_mode"`
 	OffCPUThreshold        float64       `mapstructure:"off_cpu_threshold"`
 	IncludeEnvVars         string        `mapstructure:"include_env_vars"`
-	UProbeLinks            []string      `mapstructure:"u_probe_links"`
+	ProbeLinks             []string      `mapstructure:"probe_links"`
 	LoadProbe              bool          `mapstructure:"load_probe"`
 	MapScaleFactor         uint          `mapstructure:"map_scale_factor"`
 	BPFVerifierLogLevel    uint          `mapstructure:"bpf_verifier_log_level"`

internal/controller/controller.go

@@ -88,7 +88,7 @@ func (c *Controller) Start(ctx context.Context) error {
 		ProbabilisticThreshold: c.config.ProbabilisticThreshold,
 		OffCPUThreshold:        uint32(c.config.OffCPUThreshold * float64(math.MaxUint32)),
 		IncludeEnvVars:         envVars,
-		UProbeLinks:            c.config.UProbeLinks,
+		ProbeLinks:             c.config.ProbeLinks,
 		LoadProbe:              c.config.LoadProbe,
 		ExecutableReporter:     c.config.ExecutableReporter,
 	})
@@ -118,11 +118,11 @@ func (c *Controller) Start(ctx context.Context) error {
 		log.Infof("Enabled off-cpu profiling with p=%f", c.config.OffCPUThreshold)
 	}
 
-	if len(c.config.UProbeLinks) > 0 {
-		if err := trc.AttachUProbes(c.config.UProbeLinks); err != nil {
-			return fmt.Errorf("failed to attach uprobes: %v", err)
+	if len(c.config.ProbeLinks) > 0 {
+		if err := trc.AttachProbes(c.config.ProbeLinks); err != nil {
+			return fmt.Errorf("failed to attach probes: %v", err)
 		}
-		log.Info("Attached uprobes")
+		log.Info("Attached probes")
 	}
 
 	if c.config.ProbabilisticThreshold < tracer.ProbabilisticThresholdMax {

reporter/base_reporter.go

@@ -45,7 +45,7 @@ func (b *baseReporter) ReportTraceEvent(trace *libpf.Trace, meta *samples.TraceE
 	switch meta.Origin {
 	case support.TraceOriginSampling:
 	case support.TraceOriginOffCPU:
-	case support.TraceOriginUProbe:
+	case support.TraceOriginProbe:
 	default:
 		return fmt.Errorf("skip reporting trace for %d origin: %w", meta.Origin,
 			errUnknownOrigin)

reporter/internal/pdata/generate.go

@@ -82,7 +82,7 @@ func (p *Pdata) Generate(tree samples.TraceEventsTree,
 		for _, origin := range []libpf.Origin{
 			support.TraceOriginSampling,
 			support.TraceOriginOffCPU,
-			support.TraceOriginUProbe,
+			support.TraceOriginProbe,
 		} {
 			if len(originToEvents[origin]) == 0 {
 				// Do not append empty profiles.
@@ -146,7 +146,7 @@ func (p *Pdata) setProfile(
 	case support.TraceOriginOffCPU:
 		st.SetTypeStrindex(stringSet.Add("off_cpu"))
 		st.SetUnitStrindex(stringSet.Add("nanoseconds"))
-	case support.TraceOriginUProbe:
+	case support.TraceOriginProbe:
 		st.SetTypeStrindex(stringSet.Add("events"))
 		st.SetUnitStrindex(stringSet.Add("count"))
 	default:

support/ebpf/uprobe.ebpf.c → support/ebpf/generic_probe.ebpf.c

@@ -2,9 +2,7 @@
 #include "tracemgmt.h"
 #include "types.h"
 
-// uprobe__generic serves as entry point for uprobe based profiling.
-SEC("uprobe/generic")
-int uprobe__generic(void *ctx)
+static EBPF_INLINE int probe__generic(struct pt_regs *ctx)
 {
   u64 pid_tgid = bpf_get_current_pid_tgid();
   u32 pid      = pid_tgid >> 32;
@@ -16,5 +14,19 @@ int uprobe__generic(void *ctx)
 
   u64 ts = bpf_ktime_get_ns();
 
-  return collect_trace(ctx, TRACE_UPROBE, pid, tid, ts, 0);
+  return collect_trace(ctx, TRACE_PROBE, pid, tid, ts, 0);
+}
+
+// uprobe__generic serves as entry point for uprobe based profiling.
+SEC("uprobe/generic")
+int uprobe__generic(void *ctx)
+{
+  return probe__generic((struct pt_regs *)ctx);
+}
+
+// kprobe__generic serves as entry point for kprobe based profiling.
+SEC("kprobe/generic")
+int kprobe__generic(struct pt_regs *ctx)
+{
+  return probe__generic(ctx);
 }

support/ebpf/types.h

@@ -362,7 +362,7 @@ typedef enum TraceOrigin {
   TRACE_UNKNOWN,
   TRACE_SAMPLING,
   TRACE_OFF_CPU,
-  TRACE_UPROBE,
+  TRACE_PROBE,
 } TraceOrigin;
 
 // MAX_FRAME_UNWINDS defines the maximum number of frames per

support/types_def.go

@@ -101,7 +101,7 @@ const (
 	TraceOriginUnknown  = C.TRACE_UNKNOWN
 	TraceOriginSampling = C.TRACE_SAMPLING
 	TraceOriginOffCPU   = C.TRACE_OFF_CPU
-	TraceOriginUProbe   = C.TRACE_UPROBE
+	TraceOriginProbe    = C.TRACE_PROBE
 )
 
 type ApmSpanID C.ApmSpanID

tools/probe-ctrl/cmd/probe-ctrl/main.go

@@ -4,25 +4,28 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"path/filepath"
 	"time"
 
-	"github.com/cilium/ebpf/link"
+	"go.opentelemetry.io/ebpf-profiler/tracer"
 )
 
 var (
-	argExec   string
-	argSymbol string
-	argClear  bool
+	argProbeLink string
+	argClear     bool
+	argListProgs bool
+	argBPFFS     string
 )
 
 var (
-	pinPath = "/sys/fs/bpf/probe-ctrl/"
+	defaultBPFFSPath = "/sys/fs/bpf"
 )
 
 func init() {
-	flag.StringVar(&argExec, "exec", "", "Executable to which the probe should be attached.")
-	flag.StringVar(&argSymbol, "symb", "", "Symbol in the executable to which the probe will be attached.")
+	flag.StringVar(&argProbeLink, "probe-link", "", "kprobe|kretprobe|uprobe|uretprobe:<target>[:<symbol>]")
 	flag.BoolVar(&argClear, "clear", false, "Remove probe from all links.")
+	flag.BoolVar(&argListProgs, "list", false, "List all loaded eBPF programs and exit.")
+	flag.StringVar(&argBPFFS, "bpffs", defaultBPFFSPath, "Path to BPF filesystem mount point.")
 }
 
 func main() {
@@ -32,48 +35,51 @@ func main() {
 func run() int {
 	flag.Parse()
 
+	if argListProgs {
+		return listAllPrograms()
+	}
+
+	pinPath := fmt.Sprintf("%s/probe-ctrl/", argBPFFS)
+
 	if argClear {
-		// bpf_link_detach does not exist for uprobes. So just remove
-		// the pinned path to deactivate uprobes as a work around.
 		if err := os.RemoveAll(pinPath); err != nil {
-			fmt.Fprintf(os.Stderr, "%v\n", err)
+			fmt.Fprintf(os.Stderr, "Failed to remove pin path: %v\n", err)
 			return -1
 		}
 		return 0
 	}
 
-	if argExec == "" || argSymbol == "" {
-		fmt.Fprintf(os.Stderr, "Both -exec <exec_value> and -symb <symb_value> need to be set\n")
-		return -1
-	}
-
-	exec, err := link.OpenExecutable(argExec)
+	probeSpec, err := tracer.ParseProbe(argProbeLink)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
+		fmt.Fprintf(os.Stderr, "Failed to parse probe: %v\n", err)
 		return -1
 	}
 
-	probe, err := getProbe()
+	tracerProg, err := getProbe(probeSpec.ProgName)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
+		fmt.Fprintf(os.Stderr, "Failed to get probe %q: %v\n", probeSpec.ProgName, err)
 		return -1
 	}
 
-	probeLink, err := exec.Uprobe(argSymbol, probe, nil)
+	probeLink, err := tracer.AttachProbe(tracerProg, probeSpec)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Failed attaching probe to '%s' in '%s': %v\n", argSymbol, argExec, err)
+		fmt.Fprintf(os.Stderr, "Failed to attach probe: %v\n", err)
 		return -1
 	}
 
 	if err := os.MkdirAll(pinPath, 0700); err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
+		fmt.Fprintf(os.Stderr, "Failed to create pin path: %v\n", err)
 		return -1
 	}
 
-	if err := probeLink.Pin(fmt.Sprintf("%s/%d", pinPath, time.Now().Unix())); err != nil {
+	pinFile := filepath.Join(pinPath, fmt.Sprintf("%d", time.Now().Unix()))
+	if err := probeLink.Pin(pinFile); err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to pin link: %v\n", err)
 		return -1
 	}
 
+	fmt.Printf("Attached probe %s to %s:%s\n", probeSpec.ProgName, probeSpec.Target, probeSpec.Symbol)
+	fmt.Printf("Pinned to: %s\n", pinFile)
+
 	return 0
 }

tools/probe-ctrl/cmd/probe-ctrl/probe.go

@@ -1,32 +1,71 @@
 package main
 
 import (
+	"fmt"
+
 	"github.com/cilium/ebpf"
 )
 
-func getProbe() (*ebpf.Program, error) {
+// iteratePrograms iterates through all eBPF programs and calls the visitor function for each.
+// The visitor receives the program ID and info, and should return true to continue iteration.
+func iteratePrograms(visitor func(id ebpf.ProgramID, info *ebpf.ProgramInfo) bool) {
 	var lastID ebpf.ProgramID
 
 	for {
 		nextID, err := ebpf.ProgramGetNextID(lastID)
 		if err != nil {
-			return nil, err
+			return
 		}
 		lastID = nextID
 
 		prog, err := ebpf.NewProgramFromID(nextID)
 		if err != nil {
-			return nil, err
+			continue
 		}
-		defer prog.Close()
 
 		info, err := prog.Info()
+		prog.Close()
 		if err != nil {
-			return nil, err
+			continue
 		}
 
-		if info.Name == "uprobe__generic" {
-			return prog.Clone()
+		if !visitor(nextID, info) {
+			return
 		}
 	}
 }
+
+func getProbe(name string) (*ebpf.Program, error) {
+	var found *ebpf.Program
+
+	iteratePrograms(func(id ebpf.ProgramID, info *ebpf.ProgramInfo) bool {
+		if info.Name == name {
+			prog, err := ebpf.NewProgramFromID(id)
+			if err == nil {
+				found = prog
+			}
+			return false
+		}
+		return true
+	})
+
+	if found != nil {
+		return found, nil
+	}
+	return nil, fmt.Errorf("probe %q not found", name)
+}
+
+func listAllPrograms() int {
+	fmt.Println("Loaded eBPF programs:")
+	fmt.Println("=====================")
+
+	count := 0
+	iteratePrograms(func(id ebpf.ProgramID, info *ebpf.ProgramInfo) bool {
+		count++
+		fmt.Printf("  [%d] ID=%d Name=%s Type=%s\n", count, id, info.Name, info.Type)
+		return true
+	})
+
+	fmt.Printf("\nTotal: %d programs\n", count)
+	return 0
+}

tools/probe-ctrl/go.mod

@@ -7,18 +7,51 @@ tool (
 	honnef.co/go/tools/cmd/staticcheck
 )
 
-require github.com/cilium/ebpf v0.20.0
+replace go.opentelemetry.io/ebpf-profiler => ../../
+
+require (
+	github.com/cilium/ebpf v0.20.0
+	go.opentelemetry.io/ebpf-profiler v0.0.202545
+)
 
 require (
 	github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
+	github.com/elastic/go-freelru v0.16.0 // indirect
+	github.com/elastic/go-perf v0.0.0-20241029065020-30bec95324b8 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
+	github.com/josharian/native v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/mdlayher/kobject v0.0.0-20200520190114-19ca17470d7d // indirect
+	github.com/mdlayher/netlink v1.7.2 // indirect
+	github.com/mdlayher/socket v0.4.1 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opentelemetry.io/collector/consumer v1.45.0 // indirect
+	go.opentelemetry.io/collector/consumer/xconsumer v0.139.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.45.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.45.0 // indirect
+	go.opentelemetry.io/collector/pdata/pprofile v0.139.0 // indirect
+	go.opentelemetry.io/otel v1.38.0 // indirect
+	go.opentelemetry.io/otel/metric v1.38.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/arch v0.22.0 // indirect
+	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
 	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/net v0.46.0 // indirect
 	golang.org/x/sync v0.17.0 // indirect
 	golang.org/x/sys v0.37.0 // indirect
 	golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect
+	golang.org/x/text v0.30.0 // indirect
 	golang.org/x/tools v0.38.0 // indirect
-	golang.org/x/tools/go/expect v0.1.1-deprecated // indirect
-	golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect
 	golang.org/x/vuln v1.1.4 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect
+	google.golang.org/grpc v1.76.0 // indirect
+	google.golang.org/protobuf v1.36.10 // indirect
 	honnef.co/go/tools v0.6.1 // indirect
 )