program: perform CO-RE against all kmod

CO-RE relocations are currently performed against vmlinux types only.
If a program attaches to a function in a kernel module we also
relocate against types in that module. This has the downside that
it's not possible to inspect a variable with a type defined in a
kernel module when tracing a function in vmlinux itself.

The historical reason for this is that our BTF parsing was quite
expensive. Now that we have lazy loading we can afford to relocate
against all types in the kernel.

Below is the change in performance against the last release:

    core: 1
    goos: linux
    goarch: amd64
    pkg: github.com/cilium/ebpf
    cpu: 13th Gen Intel(R) Core(TM) i7-1365U
                        │    base.txt    │              all.txt               │
                        │     sec/op     │   sec/op     vs base               │
    NewCollectionManyProgs   336.94m ± 205%   16.38m ± 2%  -95.14% (p=0.002 n=6)

                        │    base.txt     │               all.txt               │
                        │      B/op       │     B/op      vs base               │
    NewCollectionManyProgs   112.541Mi ± 45%   4.011Mi ± 0%  -96.44% (p=0.002 n=6)

                        │    base.txt    │              all.txt               │
                        │   allocs/op    │  allocs/op   vs base               │
    NewCollectionManyProgs   386.00k ± 202%   39.78k ± 0%  -89.70% (p=0.002 n=6)

Fixes: #1511

Signed-off-by: Lorenz Bauer <[email protected]>

Author: lmb

btf/core_reloc_test.go

@@ -36,7 +36,8 @@ func TestCORERelocationLoad(t *testing.T) {
 			}
 
 			prog, err := ebpf.NewProgramWithOptions(progSpec, ebpf.ProgramOptions{
-				KernelTypes: spec.Types,
+				KernelTypes:       spec.Types,
+				KernelModuleTypes: map[string]*btf.Spec{},
 			})
 			testutils.SkipIfNotSupported(t, err)
 
@@ -83,7 +84,8 @@ func TestCORERelocationRead(t *testing.T) {
 	for _, progSpec := range spec.Programs {
 		t.Run(progSpec.Name, func(t *testing.T) {
 			prog, err := ebpf.NewProgramWithOptions(progSpec, ebpf.ProgramOptions{
-				KernelTypes: targetSpec,
+				KernelTypes:       targetSpec,
+				KernelModuleTypes: map[string]*btf.Spec{},
 			})
 			testutils.SkipIfNotSupported(t, err)
 			if err != nil {
@@ -149,7 +151,8 @@ func TestCOREPoisonLineInfo(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			t.Log(progSpec.Instructions)
 			_, err := ebpf.NewProgramWithOptions(progSpec, ebpf.ProgramOptions{
-				KernelTypes: empty,
+				KernelTypes:       empty,
+				KernelModuleTypes: map[string]*btf.Spec{},
 			})
 			testutils.SkipIfNotSupported(t, err)
 

btf/kernel.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
+	"sort"
 	"sync"
 
 	"github.com/cilium/ebpf/internal"
@@ -189,7 +191,8 @@ func findVMLinux() (*os.File, error) {
 // It is not safe for concurrent use.
 type Cache struct {
 	KernelTypes   *Spec
-	KernelModules map[string]*Spec
+	ModuleTypes   map[string]*Spec
+	LoadedModules []string
 }
 
 // NewCache creates a new Cache.
@@ -214,7 +217,11 @@ func NewCache() *Cache {
 		modules[name] = &Spec{decoder: decoder}
 	}
 
-	return &Cache{kernel, modules}
+	if len(modules) == 0 {
+		return &Cache{kernel, nil, nil}
+	}
+
+	return &Cache{kernel, modules, nil}
 }
 
 // Kernel is equivalent to [LoadKernelSpec], except that repeated calls do
@@ -234,12 +241,12 @@ func (c *Cache) Kernel() (*Spec, error) {
 //
 // All modules also share the return value of [Kernel] as their base.
 func (c *Cache) Module(name string) (*Spec, error) {
-	if spec := c.KernelModules[name]; spec != nil {
+	if spec := c.ModuleTypes[name]; spec != nil {
 		return spec, nil
 	}
 
-	if c.KernelModules == nil {
-		c.KernelModules = make(map[string]*Spec)
+	if c.ModuleTypes == nil {
+		c.ModuleTypes = make(map[string]*Spec)
 	}
 
 	base, err := c.Kernel()
@@ -260,6 +267,32 @@ func (c *Cache) Module(name string) (*Spec, error) {
 	}
 
 	spec = &Spec{decoder: decoder}
-	c.KernelModules[name] = spec
+	c.ModuleTypes[name] = spec
 	return spec, err
 }
+
+// Modules returns a sorted list of all loaded modules.
+func (c *Cache) Modules() ([]string, error) {
+	if c.LoadedModules != nil {
+		return c.LoadedModules, nil
+	}
+
+	btfDir, err := os.Open("/sys/kernel/btf")
+	if err != nil {
+		return nil, err
+	}
+	defer btfDir.Close()
+
+	entries, err := btfDir.Readdirnames(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	entries = slices.DeleteFunc(entries, func(s string) bool {
+		return s == "vmlinux"
+	})
+
+	sort.Strings(entries)
+	c.LoadedModules = entries
+	return entries, nil
+}

btf/kernel_test.go

@@ -34,7 +34,8 @@ func TestCache(t *testing.T) {
 	c := NewCache()
 
 	qt.Assert(t, qt.IsNil(c.KernelTypes))
-	qt.Assert(t, qt.HasLen(c.KernelModules, 0))
+	qt.Assert(t, qt.HasLen(c.ModuleTypes, 0))
+	qt.Assert(t, qt.IsNil(c.LoadedModules))
 
 	// Test that Kernel() creates only one copy
 	spec1, err := c.Kernel()
@@ -75,7 +76,12 @@ func TestCache(t *testing.T) {
 	qt.Assert(t, qt.IsNotNil(c.KernelTypes))
 	qt.Assert(t, qt.Not(qt.Equals(c.KernelTypes, vmlinux)))
 	if testmod != nil {
-		qt.Assert(t, qt.IsNotNil(c.KernelModules["bpf_testmod"]))
-		qt.Assert(t, qt.Not(qt.Equals(c.KernelModules["bpf_testmod"], testmod)))
+		qt.Assert(t, qt.IsNotNil(c.ModuleTypes["bpf_testmod"]))
+		qt.Assert(t, qt.Not(qt.Equals(c.ModuleTypes["bpf_testmod"], testmod)))
 	}
+
+	// Test that Modules only reads modules once.
+	_, err = c.Modules()
+	qt.Assert(t, qt.IsNil(err))
+	qt.Assert(t, qt.IsNotNil(c.LoadedModules))
 }

btf/unmarshal.go

@@ -269,7 +269,9 @@ func (d *decoder) TypesByName(name essentialName) ([]Type, error) {
 	}
 
 	if len(types) == 0 {
-		return nil, fmt.Errorf("type with name %s: %w", name, ErrNotFound)
+		// Return an unwrapped error because this is on the hot path
+		// for CO-RE.
+		return nil, ErrNotFound
 	}
 
 	return types, nil

elf_reader_test.go

@@ -1130,6 +1130,7 @@ func TestLibBPFCompat(t *testing.T) {
 
 				opts := opts // copy
 				opts.Programs.KernelTypes = btfSpec
+				opts.Programs.KernelModuleTypes = map[string]*btf.Spec{}
 				load(t, spec, opts, valid)
 			})
 		}

linker.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/fs"
 	"math"
 	"slices"
 	"strings"
@@ -124,7 +123,7 @@ func hasFunctionReferences(insns asm.Instructions) bool {
 //
 // Passing a nil target will relocate against the running kernel. insns are
 // modified in place.
-func applyRelocations(insns asm.Instructions, kmodName string, bo binary.ByteOrder, b *btf.Builder, c *btf.Cache) error {
+func applyRelocations(insns asm.Instructions, bo binary.ByteOrder, b *btf.Builder, c *btf.Cache) error {
 	var relos []*btf.CORERelocation
 	var reloInsns []*asm.Instruction
 	iter := insns.Iterate()
@@ -143,22 +142,26 @@ func applyRelocations(insns asm.Instructions, kmodName string, bo binary.ByteOrd
 		bo = internal.NativeEndian
 	}
 
-	var targets []*btf.Spec
 	kernelTarget, err := c.Kernel()
 	if err != nil {
 		return fmt.Errorf("load kernel spec: %w", err)
 	}
+
+	modules, err := c.Modules()
+	if err != nil {
+		return err
+	}
+
+	targets := make([]*btf.Spec, 0, 1+len(modules))
 	targets = append(targets, kernelTarget)
 
-	if kmodName != "" {
-		kmodTarget, err := c.Module(kmodName)
-		// Ignore ErrNotExists to cater to kernels which have CONFIG_DEBUG_INFO_BTF_MODULES disabled.
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("load kernel module spec: %w", err)
-		}
-		if err == nil {
-			targets = append(targets, kmodTarget)
+	for _, kmod := range modules {
+		spec, err := c.Module(kmod)
+		if err != nil {
+			return fmt.Errorf("load BTF for kmod %s: %w", kmod, err)
 		}
+
+		targets = append(targets, spec)
 	}
 
 	fixups, err := btf.CORERelocate(relos, targets, bo, b.Add)

prog.go

@@ -8,12 +8,12 @@ import (
 	"math"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"time"
 
 	"github.com/cilium/ebpf/asm"
 	"github.com/cilium/ebpf/btf"
 	"github.com/cilium/ebpf/internal"
-	"github.com/cilium/ebpf/internal/kallsyms"
 	"github.com/cilium/ebpf/internal/linux"
 	"github.com/cilium/ebpf/internal/platform"
 	"github.com/cilium/ebpf/internal/sys"
@@ -171,17 +171,6 @@ func (ps *ProgramSpec) Tag() (string, error) {
 	return ps.Instructions.Tag(internal.NativeEndian)
 }
 
-// kernelModule returns the kernel module providing the symbol in
-// ProgramSpec.AttachTo, if any. Returns an empty string if the symbol is not
-// present or not part of a kernel module.
-func (ps *ProgramSpec) kernelModule() (string, error) {
-	if ps.targetsKernelModule() {
-		return kallsyms.Module(ps.AttachTo)
-	}
-
-	return "", nil
-}
-
 // targetsKernelModule returns true if the program supports being attached to a
 // symbol provided by a kernel module.
 func (ps *ProgramSpec) targetsKernelModule() bool {
@@ -303,13 +292,8 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache)
 	insns := make(asm.Instructions, len(spec.Instructions))
 	copy(insns, spec.Instructions)
 
-	kmodName, err := spec.kernelModule()
-	if err != nil {
-		return nil, fmt.Errorf("kernel module search: %w", err)
-	}
-
 	var b btf.Builder
-	if err := applyRelocations(insns, kmodName, spec.ByteOrder, &b, c); err != nil {
+	if err := applyRelocations(insns, spec.ByteOrder, &b, c); err != nil {
 		return nil, fmt.Errorf("apply CO-RE relocations: %w", err)
 	}
 
@@ -1212,7 +1196,14 @@ func newBTFCache(opts *ProgramOptions) *btf.Cache {
 	c := btf.NewCache()
 	if opts.KernelTypes != nil {
 		c.KernelTypes = opts.KernelTypes
-		c.KernelModules = opts.KernelModuleTypes
+		c.ModuleTypes = opts.KernelModuleTypes
+		if opts.KernelModuleTypes != nil {
+			c.LoadedModules = []string{}
+			for name := range opts.KernelModuleTypes {
+				c.LoadedModules = append(c.LoadedModules, name)
+			}
+			sort.Strings(c.LoadedModules)
+		}
 	}
 	return c
 }

prog_test.go

@@ -881,7 +881,8 @@ func TestProgramLoadErrors(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			t.Log(progSpec.Instructions)
 			_, err := newProgram(t, progSpec, &ProgramOptions{
-				KernelTypes: empty,
+				KernelTypes:       empty,
+				KernelModuleTypes: map[string]*btf.Spec{},
 			})
 			testutils.SkipIfNotSupported(t, err)
 
@@ -902,15 +903,6 @@ func TestProgramTargetsKernelModule(t *testing.T) {
 	qt.Assert(t, qt.IsTrue(ps.targetsKernelModule()))
 }
 
-func TestProgramAttachToKernelModule(t *testing.T) {
-	requireTestmod(t)
-
-	ps := ProgramSpec{AttachTo: "bpf_testmod_test_read", Type: Tracing, AttachType: AttachTraceFEntry}
-	mod, err := ps.kernelModule()
-	qt.Assert(t, qt.IsNil(err))
-	qt.Assert(t, qt.Equals(mod, "bpf_testmod"))
-}
-
 func BenchmarkNewProgram(b *testing.B) {
 	testutils.SkipOnOldKernel(b, "5.18", "kfunc support")
 	spec, err := LoadCollectionSpec(testutils.NativeFile(b, "testdata/kfunc-%s.elf"))