Fix QEMU distro tests, switch to hash spec_id map and remove pid based attachment

.github/workflows/unit-test-on-pull-request.yml

@@ -246,15 +246,27 @@ jobs:
           sudo go test ./interpreter/... -v -run "TestIntegration/(node-local-nightly|node-latest)"
 
   distro-qemu-tests:
-    name: Full distro QEMU tests (kernel ${{ matrix.kernel }})
+    name: Distro QEMU tests (${{ matrix.kernel }} ${{ matrix.target_arch }})
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     strategy:
       matrix:
-        kernel:
-          #- 5.10.217  # 5.10 doesn't have bpf cookies
-          - 5.15.159
-          - 6.8.10    # Post-6.6, supports multi-uprobe
+        include:
+          - { target_arch: amd64, kernel: 5.4.276 }
+          - { target_arch: amd64, kernel: 5.10.217 }
+          - { target_arch: amd64, kernel: 5.15.159 }
+          - { target_arch: amd64, kernel: 6.1.91 }
+          - { target_arch: amd64, kernel: 6.6.31 }
+          - { target_arch: amd64, kernel: 6.8.10 }
+          - { target_arch: amd64, kernel: 6.9.1 }
+          - { target_arch: amd64, kernel: 6.12.16 }
+          - { target_arch: amd64, kernel: 6.16 }
+
+          # ARM64 (NOTE: older ARM64 kernels are not available in Cilium repos)
+          - { target_arch: arm64, kernel: 6.6.31 }
+          - { target_arch: arm64, kernel: 6.8.4 }
+          - { target_arch: arm64, kernel: 6.9.1 }
+          - { target_arch: arm64, kernel: 6.12.16 }
     steps:
       - name: Clone code
         uses: actions/checkout@v4
@@ -263,15 +275,32 @@ jobs:
         with:
           go-version-file: go.mod
           cache-dependency-path: go.sum
+      - name: Set up environment
+        uses: ./.github/workflows/env
       - name: Install dependencies
         run: |
           sudo apt-get update -y
-          sudo apt-get install -y qemu-system-x86 debootstrap systemtap-sdt-dev
+          case "${{ matrix.target_arch }}" in
+            amd64) sudo apt-get -y install qemu-system-x86;;
+            arm64) sudo apt-get -y install qemu-system-arm;;
+            *) echo >&2 "bug: bad arch selected"; exit 1;;
+          esac
+          sudo apt-get install -y debootstrap systemtap-sdt-dev
       - name: Download kernel
         run: |
           cd test/distro-qemu
+          case "${{ matrix.target_arch }}" in
+            amd64) export QEMU_ARCH=x86_64;;
+            arm64) export QEMU_ARCH=aarch64;;
+            *) echo >&2 "bug: bad arch selected"; exit 1;;
+          esac
           ./download-kernel.sh ${{ matrix.kernel }}
-      - name: Run RTLD tests in QEMU
+      - name: Run Full Distro tests in QEMU
         run: |
           cd test/distro-qemu
+          case "${{ matrix.target_arch }}" in
+            amd64) export QEMU_ARCH=x86_64;;
+            arm64) export QEMU_ARCH=aarch64;;
+            *) echo >&2 "bug: bad arch selected"; exit 1;;
+          esac
           ./build-and-run.sh ${{ matrix.kernel }}

interpreter/gpu/cuda.go

@@ -50,7 +50,6 @@ type data struct {
 type Instance struct {
 	interpreter.InstanceStubs
 	path string
-	link interpreter.LinkCloser
 	pid  libpf.PID
 }
 
@@ -68,9 +67,9 @@ func Loader(ebpf interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interpr
 	if err != nil {
 		return nil, err
 	}
+
 	// We use the existence of the .note.stapsdt section to determine if this is a
-	// process that has libparcagpucupti.so loaded. Its cheaper and more reliable than loading
-	// the symbol table.
+	// process that has libparcagpucupti.so loaded.
 	probes, err := ef.ParseUSDTProbes()
 	if err != nil {
 		return nil, err
@@ -96,7 +95,6 @@ func Loader(ebpf interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interpr
 	return nil, nil
 }
 
-
 func (d *data) Attach(ebpf interpreter.EbpfHandler, pid libpf.PID, _ libpf.Address,
 	_ remotememory.RemoteMemory) (interpreter.Instance, error) {
 	// Maps usdt probe name to ebpf program name.
@@ -115,12 +113,19 @@ func (d *data) Attach(ebpf interpreter.EbpfHandler, pid libpf.PID, _ libpf.Addre
 			progNames[i] = "usdt_parcagpu_cuda_kernel"
 		}
 	}
-	lc, err := ebpf.AttachUSDTProbes(pid, d.path, "cuda_probe", d.probes, cookies, progNames, true)
-	if err != nil {
-		return nil, err
+
+	var lc interpreter.LinkCloser
+	if d.link == nil {
+		var err error
+		lc, err = ebpf.AttachUSDTProbes(pid, d.path, "cuda_probe", d.probes, cookies, progNames)
+		if err != nil {
+			return nil, err
+		}
+		log.Debugf("[cuda] parcagpu USDT probes attached for %s", d.path)
+		d.link = lc
+	} else {
+		log.Debugf("[cuda] parcagpu USDT probes already attached for %s", d.path)
 	}
-	log.Debugf("[cuda] parcagpu USDT probes attached for %s", d.path)
-	d.link = lc
 
 	// Create and register fixer for this PID
 	fixer := &gpuTraceFixer{
@@ -129,24 +134,14 @@ func (d *data) Attach(ebpf interpreter.EbpfHandler, pid libpf.PID, _ libpf.Addre
 	}
 
 	gpuFixers.Store(pid, fixer)
-
 	return &Instance{
-		link: lc,
 		path: d.path,
 		pid:  pid,
 	}, nil
 }
 
-// Detach removes the fixer for this PID and closes the link if needed.
 func (i *Instance) Detach(_ interpreter.EbpfHandler, _ libpf.PID) error {
 	gpuFixers.Delete(i.pid)
-
-	if i.link != nil {
-		log.Debugf("[cuda] parcagpu USDT probes closed for %s", i.path)
-		if err := i.link.Detach(); err != nil {
-			return err
-		}
-	}
 	return nil
 }
 

interpreter/instancestubs.go

@@ -73,7 +73,7 @@ func (m *EbpfHandlerStubs) DeleteProcData(libpf.InterpreterType, libpf.PID) erro
 }
 
 func (mockup *EbpfHandlerStubs) AttachUSDTProbes(libpf.PID, string, string, []pfelf.USDTProbe,
-	[]uint64, []string, bool) (LinkCloser, error) {
+	[]uint64, []string) (LinkCloser, error) {
 	return nil, nil
 }
 

interpreter/rtld/rtld.go

@@ -22,7 +22,6 @@ type data struct {
 // instance represents a per-PID instance of the dlopen interpreter
 type instance struct {
 	interpreter.InstanceStubs
-	lc interpreter.LinkCloser
 }
 
 // Loader detects if the ELF file contains the dlopen symbol in its dynamic symbol table
@@ -37,7 +36,6 @@ func Loader(ebpf interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interpr
 	// Look for the dlopen symbol in the dynamic symbol table
 	sym, err := ef.LookupSymbol("dlopen")
 	if err != nil || sym == nil {
-		// No dlopen symbol found, this library doesn't support dynamic loading
 		return nil, nil
 	}
 
@@ -52,26 +50,21 @@ func Loader(ebpf interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interpr
 // Attach attaches the uprobe to the dlopen function
 func (d *data) Attach(ebpf interpreter.EbpfHandler, pid libpf.PID, bias libpf.Address,
 	_ remotememory.RemoteMemory) (interpreter.Instance, error) {
-	// Attach uprobe to dlopen using the address stored during Loader
-	lc, err := ebpf.AttachUprobe(pid, d.path, d.address, "uprobe_dlopen")
-	if err != nil {
-		return nil, fmt.Errorf("failed to attach uprobe to dlopen: %w", err)
+	var lc interpreter.LinkCloser
+	if d.lc == nil {
+		// Attach uprobe to dlopen using the address stored during Loader
+		var err error
+		lc, err = ebpf.AttachUprobe(pid, d.path, d.address, "uprobe_dlopen")
+		if err != nil {
+			return nil, fmt.Errorf("failed to attach uprobe to dlopen: %w", err)
+		}
+		d.lc = lc
 	}
 
 	log.Debugf("[dlopen] Attached uprobe to dlopen for PID %d on %s at 0x%x",
 		pid, d.path, d.address)
 
-	d.lc = lc
-	return &instance{lc: lc}, nil
-}
-
-// Detach removes the uprobe
-func (i *instance) Detach(_ interpreter.EbpfHandler, pid libpf.PID) error {
-	log.Debugf("[dlopen] Detach called for PID %d", pid)
-	if i.lc != nil {
-		return i.lc.Detach()
-	}
-	return nil
+	return &instance{}, nil
 }
 
 // Unload cleans up the uprobe link
@@ -80,7 +73,6 @@ func (d *data) Unload(_ interpreter.EbpfHandler) {
 		if err := d.lc.Unload(); err != nil {
 			log.Errorf("[dlopen] Failed to unload uprobe link: %v", err)
 		}
-		d.lc = nil
 	}
 	log.Debugf("[dlopen] Unloaded uprobe for %s", d.path)
 }

interpreter/rtld/rtld_test.go

@@ -1,8 +1,6 @@
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
-//go:build amd64 && !integration
-
 package rtld_test
 
 import (
@@ -15,6 +13,7 @@ import (
 	"github.com/coreos/pkg/dlopen"
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/ebpf-profiler/libpf"
 	"go.opentelemetry.io/ebpf-profiler/metrics"
 	"go.opentelemetry.io/ebpf-profiler/support"
 	"go.opentelemetry.io/ebpf-profiler/testutils"
@@ -23,22 +22,32 @@ import (
 	"go.opentelemetry.io/ebpf-profiler/util"
 )
 
-func TestIntegration(t *testing.T) {
+func test(t *testing.T) {
 	if !testutils.IsRoot() {
 		t.Skip("This test requires root privileges")
 	}
 
+	// Enable debug logging for CI debugging
+	if os.Getenv("DEBUG_TEST") != "" {
+		log.SetLevel(log.DebugLevel)
+	}
+
 	// Create a context for the tracer
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
 	// Start the tracer with all tracers enabled
 	traceCh, trc := testutils.StartTracer(ctx, t,
-		tracertypes.AllTracers(),
+		tracertypes.IncludedTracers(0),
 		&testutils.MockReporter{},
 		false)
 	defer trc.Close()
 
+	trc.StartPIDEventProcessor(ctx)
+
+	// tickle tihs process to speed things up
+	trc.ForceProcessPID(libpf.PID(uint32(os.Getpid())))
+
 	// Consume traces to prevent blocking
 	go func() {
 		for {
@@ -73,70 +82,20 @@ func TestIntegration(t *testing.T) {
 
 		// Check that the metric was incremented
 		return finalCount > initialCount
-	}, 10*time.Second, 50*time.Millisecond)
+	}, 10*time.Second, 100*time.Millisecond)
 }
 
-func TestIntegrationSingleShot(t *testing.T) {
-	if !testutils.IsRoot() {
-		t.Skip("This test requires root privileges")
-	}
-
-	// Enable debug logging for CI debugging
-	if os.Getenv("DEBUG_TEST") != "" {
-		log.SetLevel(log.DebugLevel)
-	}
+func TestIntegration(t *testing.T) {
+	test(t)
+}
 
-	// Override HasMultiUprobeSupport to force single-shot mode
+func TestIntegrationSingleShot(t *testing.T) {
+	// Override HasMultiUprobeSupport to force single-shot mode on newer kernels.
 	multiUProbeOverride := false
 	util.SetTestOnlyMultiUprobeSupport(&multiUProbeOverride)
 	defer util.SetTestOnlyMultiUprobeSupport(nil)
 
-	// Create a context for the tracer
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	// Start the tracer with all tracers enabled
-	traceCh, trc := testutils.StartTracer(ctx, t,
-		tracertypes.AllTracers(),
-		&testutils.MockReporter{},
-		false)
-	defer trc.Close()
-
-	// Consume traces to prevent blocking
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-traceCh:
-				// Discard traces
-			}
-		}
-	}()
-
-	// retry a few times to get the metric, our process has to be detected and
-	// the dlopen uprobe has to attach.
-	require.Eventually(t, func() bool {
-		// Get the initial metric value
-		initialCount := getEBPFMetricValue(trc, metrics.IDDlopenUprobeHits)
-		//t.Logf("Initial dlopen uprobe metric count: %d", initialCount)
-
-		// Use dlopen to load a shared library
-		// libm is a standard math library that's always present
-		lib, err := dlopen.GetHandle([]string{
-			"/lib/x86_64-linux-gnu/libm.so.6",
-			"libm.so.6",
-		})
-		require.NoError(t, err, "Failed to open libm.so.6")
-		defer lib.Close()
-
-		// Get the metrics after dlopen
-		finalCount := getEBPFMetricValue(trc, metrics.IDDlopenUprobeHits)
-		//t.Logf("Final dlopen uprobe metric count: %d", finalCount)
-
-		// Check that the metric was incremented
-		return finalCount > initialCount
-	}, 10*time.Second, 50*time.Millisecond)
+	test(t)
 }
 
 func getEBPFMetricValue(trc *tracer.Tracer, metricID metrics.MetricID) uint64 {

interpreter/types.go

@@ -117,17 +117,13 @@ type EbpfHandler interface {
 	// AttachUSDTProbes attaches an eBPF program to USDT probes in the specified binary.
 	//
 	// Parameters:
-	//  - pid: The process ID. Required for older kernels (pre-6.6) that cannot attach to shared
-	//    libraries without a PID. On newer kernels with multi-uprobe support, this is ignored
-	//    when probeAll is true.
+	//  - pid: The process ID. Required for getting path to exe via procfs.
 	//  - path: Full path to the binary containing the USDT probes.
 	//  - multiProgName: Name of eBPF program to use for multi-uprobe attachment (newer kernels).
 	//  - probes: The USDT probe definitions to attach to.
 	//  - cookies: Optional cookies to pass to the eBPF program (one per probe, or nil).
 	//  - singleProgNames: eBPF program names for single-shot attachment (older kernels, one
 	//    per probe).
-	//  - probeAll: If true and the kernel supports it, attach to all processes using this
-	//    binary. If false, only attach to the specified pid.
 	//
 	// Returns:
 	//  - LinkCloser: A handle to the attached probes. The caller must:
@@ -136,14 +132,13 @@ type EbpfHandler interface {
 	//    2. Call LinkCloser.Detach() from Instance.Detach() to detach from the specific PID
 	//    3. Call LinkCloser.Unload() from Data.Unload() to fully clean up the eBPF program
 	AttachUSDTProbes(pid libpf.PID, path, multiProgName string, probes []pfelf.USDTProbe,
-		cookies []uint64, singleProgNames []string, probeAll bool) (LinkCloser, error)
+		cookies []uint64, singleProgNames []string) (LinkCloser, error)
 
 	// AttachUprobe attaches an eBPF uprobe to a function at a specific offset in a binary
 	AttachUprobe(pid libpf.PID, path string, offset uint64, progName string) (LinkCloser, error)
 }
 
 type LinkCloser interface {
-	Detach() error
 	Unload() error
 }