Fix QEMU distro tests, switch to hash spec_id map and remove pid based attachment

.github/workflows/unit-test-on-pull-request.yml

@@ -246,15 +246,27 @@ jobs:
           sudo go test ./interpreter/... -v -run "TestIntegration/(node-local-nightly|node-latest)"
 
   distro-qemu-tests:
-    name: Full distro QEMU tests (kernel ${{ matrix.kernel }})
+    name: Distro QEMU tests (${{ matrix.kernel }} ${{ matrix.target_arch }})
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     strategy:
       matrix:
-        kernel:
-          #- 5.10.217  # 5.10 doesn't have bpf cookies
-          - 5.15.159
-          - 6.8.10    # Post-6.6, supports multi-uprobe
+        include:
+          - { target_arch: amd64, kernel: 5.4.276 }
+          - { target_arch: amd64, kernel: 5.10.217 }
+          - { target_arch: amd64, kernel: 5.15.159 }
+          - { target_arch: amd64, kernel: 6.1.91 }
+          - { target_arch: amd64, kernel: 6.6.31 }
+          - { target_arch: amd64, kernel: 6.8.10 }
+          - { target_arch: amd64, kernel: 6.9.1 }
+          - { target_arch: amd64, kernel: 6.12.16 }
+          - { target_arch: amd64, kernel: 6.16 }
+
+          # ARM64 (NOTE: older ARM64 kernels are not available in Cilium repos)
+          - { target_arch: arm64, kernel: 6.6.31 }
+          - { target_arch: arm64, kernel: 6.8.4 }
+          - { target_arch: arm64, kernel: 6.9.1 }
+          - { target_arch: arm64, kernel: 6.12.16 }
     steps:
       - name: Clone code
         uses: actions/checkout@v4
@@ -263,15 +275,32 @@ jobs:
         with:
           go-version-file: go.mod
           cache-dependency-path: go.sum
+      - name: Set up environment
+        uses: ./.github/workflows/env
       - name: Install dependencies
         run: |
           sudo apt-get update -y
-          sudo apt-get install -y qemu-system-x86 debootstrap systemtap-sdt-dev
+          case "${{ matrix.target_arch }}" in
+            amd64) sudo apt-get -y install qemu-system-x86;;
+            arm64) sudo apt-get -y install qemu-system-arm;;
+            *) echo >&2 "bug: bad arch selected"; exit 1;;
+          esac
+          sudo apt-get install -y debootstrap systemtap-sdt-dev
       - name: Download kernel
         run: |
           cd test/distro-qemu
+          case "${{ matrix.target_arch }}" in
+            amd64) export QEMU_ARCH=x86_64;;
+            arm64) export QEMU_ARCH=aarch64;;
+            *) echo >&2 "bug: bad arch selected"; exit 1;;
+          esac
           ./download-kernel.sh ${{ matrix.kernel }}
-      - name: Run RTLD tests in QEMU
+      - name: Run Full Distro tests in QEMU
         run: |
           cd test/distro-qemu
+          case "${{ matrix.target_arch }}" in
+            amd64) export QEMU_ARCH=x86_64;;
+            arm64) export QEMU_ARCH=aarch64;;
+            *) echo >&2 "bug: bad arch selected"; exit 1;;
+          esac
           ./build-and-run.sh ${{ matrix.kernel }}

interpreter/rtld/rtld_test.go

@@ -1,8 +1,6 @@
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
-//go:build amd64 && !integration
-
 package rtld_test
 
 import (
@@ -15,6 +13,7 @@ import (
 	"github.com/coreos/pkg/dlopen"
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/ebpf-profiler/libpf"
 	"go.opentelemetry.io/ebpf-profiler/metrics"
 	"go.opentelemetry.io/ebpf-profiler/support"
 	"go.opentelemetry.io/ebpf-profiler/testutils"
@@ -23,22 +22,32 @@ import (
 	"go.opentelemetry.io/ebpf-profiler/util"
 )
 
-func TestIntegration(t *testing.T) {
+func test(t *testing.T) {
 	if !testutils.IsRoot() {
 		t.Skip("This test requires root privileges")
 	}
 
+	// Enable debug logging for CI debugging
+	if os.Getenv("DEBUG_TEST") != "" {
+		log.SetLevel(log.DebugLevel)
+	}
+
 	// Create a context for the tracer
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
 	// Start the tracer with all tracers enabled
 	traceCh, trc := testutils.StartTracer(ctx, t,
-		tracertypes.AllTracers(),
+		tracertypes.IncludedTracers(0),
 		&testutils.MockReporter{},
 		false)
 	defer trc.Close()
 
+	trc.StartPIDEventProcessor(ctx)
+
+	// tickle tihs process to speed things up
+	trc.ForceProcessPID(libpf.PID(uint32(os.Getpid())))
+
 	// Consume traces to prevent blocking
 	go func() {
 		for {
@@ -73,70 +82,20 @@ func TestIntegration(t *testing.T) {
 
 		// Check that the metric was incremented
 		return finalCount > initialCount
-	}, 10*time.Second, 50*time.Millisecond)
+	}, 10*time.Second, 100*time.Millisecond)
 }
 
-func TestIntegrationSingleShot(t *testing.T) {
-	if !testutils.IsRoot() {
-		t.Skip("This test requires root privileges")
-	}
-
-	// Enable debug logging for CI debugging
-	if os.Getenv("DEBUG_TEST") != "" {
-		log.SetLevel(log.DebugLevel)
-	}
+func TestIntegration(t *testing.T) {
+	test(t)
+}
 
-	// Override HasMultiUprobeSupport to force single-shot mode
+func TestIntegrationSingleShot(t *testing.T) {
+	// Override HasMultiUprobeSupport to force single-shot mode on newer kernels.
 	multiUProbeOverride := false
 	util.SetTestOnlyMultiUprobeSupport(&multiUProbeOverride)
 	defer util.SetTestOnlyMultiUprobeSupport(nil)
 
-	// Create a context for the tracer
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	// Start the tracer with all tracers enabled
-	traceCh, trc := testutils.StartTracer(ctx, t,
-		tracertypes.AllTracers(),
-		&testutils.MockReporter{},
-		false)
-	defer trc.Close()
-
-	// Consume traces to prevent blocking
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-traceCh:
-				// Discard traces
-			}
-		}
-	}()
-
-	// retry a few times to get the metric, our process has to be detected and
-	// the dlopen uprobe has to attach.
-	require.Eventually(t, func() bool {
-		// Get the initial metric value
-		initialCount := getEBPFMetricValue(trc, metrics.IDDlopenUprobeHits)
-		//t.Logf("Initial dlopen uprobe metric count: %d", initialCount)
-
-		// Use dlopen to load a shared library
-		// libm is a standard math library that's always present
-		lib, err := dlopen.GetHandle([]string{
-			"/lib/x86_64-linux-gnu/libm.so.6",
-			"libm.so.6",
-		})
-		require.NoError(t, err, "Failed to open libm.so.6")
-		defer lib.Close()
-
-		// Get the metrics after dlopen
-		finalCount := getEBPFMetricValue(trc, metrics.IDDlopenUprobeHits)
-		//t.Logf("Final dlopen uprobe metric count: %d", finalCount)
-
-		// Check that the metric was incremented
-		return finalCount > initialCount
-	}, 10*time.Second, 50*time.Millisecond)
+	test(t)
 }
 
 func getEBPFMetricValue(trc *tracer.Tracer, metricID metrics.MetricID) uint64 {

support/usdt/test/usdt_integration_test.go

@@ -21,6 +21,7 @@ import (
 	"go.opentelemetry.io/ebpf-profiler/reporter"
 	"go.opentelemetry.io/ebpf-profiler/tracer"
 	tracertypes "go.opentelemetry.io/ebpf-profiler/tracer/types"
+	"go.opentelemetry.io/ebpf-profiler/util"
 )
 
 type mockIntervals struct{}
@@ -36,15 +37,15 @@ func (mockReporter) ExecutableMetadata(_ *reporter.ExecutableMetadataArgs) {}
 
 // testSetup encapsulates all the common test setup
 type testSetup struct {
-	t            *testing.T
-	testBinary   string
-	testProbes   map[string]pfelf.USDTProbe
-	probeList    []pfelf.USDTProbe
-	tracer       *tracer.Tracer
-	ebpfHandler  interpreter.EbpfHandler
-	resultsMap   *cebpf.Map
-	ctx          context.Context
-	cancelFunc   context.CancelFunc
+	t           *testing.T
+	testBinary  string
+	testProbes  map[string]pfelf.USDTProbe
+	probeList   []pfelf.USDTProbe
+	tracer      *tracer.Tracer
+	ebpfHandler interpreter.EbpfHandler
+	resultsMap  *cebpf.Map
+	ctx         context.Context
+	cancelFunc  context.CancelFunc
 }
 
 // setupTest performs all common initialization for USDT integration tests
@@ -53,6 +54,10 @@ func setupTest(t *testing.T) *testSetup {
 		t.Skip("This test requires root privileges to load eBPF programs")
 	}
 
+	if !util.HasBpfGetAttachCookie() {
+		t.Skip("This test requires kernel support for bpf_get_attach_cookie")
+	}
+
 	// Get the test binary path
 	testBinary, err := os.Executable()
 	if err != nil {
@@ -218,14 +223,14 @@ func TestUSDTProbeWithEBPFSingle(t *testing.T) {
 
 	// Individual program names for each probe
 	progNames := []string{
-		"usdt_simple_probe",
-		"usdt_memory_probe",
-		"usdt_const_probe",
-		"usdt_mixed_probe",
-		"usdt_int32_args",
-		"usdt_int64_args",
-		"usdt_mixed_refs",
-		"usdt_uint8_args",
+		"simple_probe",
+		"memory_probe",
+		"const_probe",
+		"mixed_probe",
+		"int32_args",
+		"int64_args",
+		"mixed_refs",
+		"uint8_args",
 	}
 
 	// Attach USDT probes with individual programs
@@ -257,6 +262,10 @@ func TestUSDTProbeWithEBPFSingle(t *testing.T) {
 // TestUSDTProbeWithEBPFMulti tests USDT probes using multi-probe attachment with cookies.
 // This mimics how CUDA probes work: one multi-probe program that dispatches based on cookie.
 func TestUSDTProbeWithEBPFMulti(t *testing.T) {
+	if !util.HasMultiUprobeSupport() {
+		t.Skip("This test requires kernel support for uprobe multi-attach")
+	}
+
 	setup := setupTest(t)
 	defer setup.cleanup()
 

test/distro-qemu/build-and-run.sh

@@ -14,6 +14,16 @@ CACHE_DIR="${CACHE_DIR:-/tmp/debootstrap-cache}"
 echo "Building rootfs with $DISTRO $RELEASE..."
 
 # Clean up previous builds
+# First, unmount any leftover mounts from previous debootstrap runs
+if [ -d "$ROOTFS_DIR" ]; then
+    echo "Cleaning up any mounted filesystems in $ROOTFS_DIR..."
+    # Find all mount points under ROOTFS_DIR and unmount them in reverse order (deepest first)
+    findmnt -o TARGET -n -l | grep "^$(pwd)/$ROOTFS_DIR" | sort -r | while read -r mountpoint; do
+        echo "  Unmounting $mountpoint"
+        sudo umount "$mountpoint" || sudo umount -l "$mountpoint" || true
+    done
+fi
+
 sudo rm -rf "$ROOTFS_DIR" "$OUTPUT_DIR"
 mkdir -p "$ROOTFS_DIR" "$OUTPUT_DIR" "$CACHE_DIR"
 
@@ -45,7 +55,7 @@ echo "Running debootstrap to create $DISTRO $RELEASE rootfs for $DEBOOTSTRAP_ARC
 sudo debootstrap --variant=minbase \
     --arch="$DEBOOTSTRAP_ARCH" \
     --cache-dir="$CACHE_DIR" \
-    "$RELEASE" "$ROOTFS_DIR" "$MIRROR"
+    "$RELEASE" "$ROOTFS_DIR" "$MIRROR" || cat "$ROOTFS_DIR/debootstrap/debootstrap.log"
 
 # Change ownership of rootfs to current user to avoid needing sudo for subsequent operations
 sudo chown -R "$(id -u):$(id -g)" "$ROOTFS_DIR"
@@ -85,11 +95,16 @@ if [[ "${USE_DOCKER}" == "1" ]] && command -v docker &> /dev/null; then
                  wget -q https://go.dev/dl/go1.24.7.linux-${GOARCH}.tar.gz && \
                  tar -C /usr/local -xzf go1.24.7.linux-${GOARCH}.tar.gz && \
                  export PATH=/usr/local/go/bin:\$PATH && \
-                 CGO_ENABLED=1 go test -c ../../interpreter/rtld ../../support/usdt"
+                 CGO_ENABLED=1 go test -c ../../interpreter/rtld ../../support/usdt/test"
 else
     # Local build with cross-compilation if needed
     echo "Building locally for ${GOARCH}..."
-    CGO_ENABLED=1 GOARCH=${GOARCH} go test -c ../../interpreter/rtld ../../support/usdt
+    if [ "$GOARCH" = "arm64" ]; then
+        # Cross-compile for ARM64 using aarch64-linux-gnu-gcc
+        CGO_ENABLED=1 GOARCH=${GOARCH} CC=aarch64-linux-gnu-gcc go test -c ../../interpreter/rtld ../../support/usdt/test
+    else
+        CGO_ENABLED=1 GOARCH=${GOARCH} go test -c ../../interpreter/rtld ../../support/usdt/test
+    fi
 fi
 
 # Copy test binary into rootfs
@@ -129,7 +144,7 @@ export DEBUG_TEST=1
 
 # Run the tests
 echo ""
-/rtld.test -test.v && /usdt.test -test.v
+/rtld.test -test.v && /test.test -test.v
 RESULT=$?
 
 if [ $RESULT -eq 0 ]; then
@@ -204,7 +219,8 @@ echo ""
 echo "===== Starting QEMU with kernel ${KERNEL_VERSION} on ${QEMU_ARCH} ====="
 echo ""
 
-# Run QEMU
+# Run QEMU and capture output
+QEMU_OUTPUT=$(mktemp)
 ${sudo} qemu-system-${QEMU_ARCH} ${additionalQemuArgs} \
     -nographic \
     -monitor none \
@@ -214,15 +230,28 @@ ${sudo} qemu-system-${QEMU_ARCH} ${additionalQemuArgs} \
     -initrd "$OUTPUT_DIR/initramfs.gz" \
     -append "${CONSOLE_ARG} init=/init quiet loglevel=3" \
     -no-reboot \
-    -display none
-
-EXIT_CODE=$?
+    -display none \
+    | tee "$QEMU_OUTPUT"
 
-# QEMU with sysrq poweroff returns 0 on clean shutdown
-if [ $EXIT_CODE -eq 0 ]; then
+# Parse output for test result
+if grep -q "===== TEST PASSED =====" "$QEMU_OUTPUT"; then
+    rm -f "$QEMU_OUTPUT"
+    echo ""
     echo "✅ Test completed successfully"
     exit 0
+elif grep -q "===== TEST FAILED" "$QEMU_OUTPUT"; then
+    rm -f "$QEMU_OUTPUT"
+    echo ""
+    echo "❌ Test failed"
+    exit 1
+elif grep -q "===== TEST TIMED OUT =====" "$QEMU_OUTPUT"; then
+    rm -f "$QEMU_OUTPUT"
+    echo ""
+    echo "❌ Test timed out"
+    exit 124
 else
-    echo "❌ Test failed with QEMU exit code $EXIT_CODE"
-    exit $EXIT_CODE
+    rm -f "$QEMU_OUTPUT"
+    echo ""
+    echo "❌ Could not determine test result (QEMU may have crashed)"
+    exit 2
 fi

tracer/tracer.go

@@ -1204,3 +1204,9 @@ func (t *Tracer) GetEbpfHandler() interpreter.EbpfHandler {
 func (t *Tracer) GetInterpretersForPID(pid libpf.PID) []interpreter.Instance {
 	return t.processManager.GetInterpretersForPID(pid)
 }
+
+// ForceProcessPID forces processing of the given PID by sending it to the
+// pidEvents channel. Used to speed up tests.
+func (t *Tracer) ForceProcessPID(pid libpf.PID) {
+	t.pidEvents <- libpf.PIDTID(uint64(pid) + uint64(pid)<<32)
+}