diff --git a/collection.go b/collection.go index ba7c417d0..974452aba 100644 --- a/collection.go +++ b/collection.go @@ -6,6 +6,7 @@ import ( "fmt" "path/filepath" "reflect" + "runtime" "strings" "github.com/cilium/ebpf/asm" @@ -512,7 +513,7 @@ func (cl *collectionLoader) loadMap(mapName string) (*Map, error) { return m, nil } - m, err := newMapWithOptions(mapSpec, cl.opts.Maps) + m, err := newMapWithOptions(mapSpec, cl.opts.Maps, cl.types) if err != nil { return nil, fmt.Errorf("map %s: %w", mapName, err) } @@ -581,6 +582,7 @@ func (cl *collectionLoader) loadProgram(progName string) (*Program, error) { } cl.programs[progName] = prog + return prog, nil } @@ -688,6 +690,40 @@ func (cl *collectionLoader) populateDeferredMaps() error { } } + if mapSpec.Type == StructOpsMap { + valueType, ok := btf.As[*btf.Struct](mapSpec.Value) + if !ok { + return fmt.Errorf("value should be a *Struct") + } + + userData, ok := mapSpec.Contents[0].Value.([]byte) + if !ok { + return fmt.Errorf("value should be an array of byte") + } + + target := btf.Type((*btf.Struct)(nil)) + _, module, err := findTargetInKernel(valueType.Name, &target, cl.types) + if err != nil { + return fmt.Errorf("lookup value type %q: %w", valueType.Name, err) + } + defer module.Close() + + vType, _ := btf.As[*btf.Struct](target) + + kernVData, err := translateStructData(userData, vType) + if err != nil { + return err + } + + if err := populateStructOpsPrograms(vType, kernVData, cl.programs); err != nil { + return err + } + defer runtime.KeepAlive(cl.programs) + + mapSpec.Contents[0] = + MapKV{Key: uint32(0), Value: kernVData} + } + // Populate and freeze the map if specified. if err := m.finalize(mapSpec); err != nil { return fmt.Errorf("populating map %s: %w", mapName, err) @@ -697,6 +733,104 @@ func (cl *collectionLoader) populateDeferredMaps() error { return nil } +// findInnerStruct returns the "inner" struct inside a value struct_ops type. +// +// Given a value like: +// +// struct bpf_struct_ops_bpf_testmod_ops { +// struct bpf_struct_ops_common common; +// struct bpf_testmod_ops data; +// }; +// +// this function returns the *btf.Struct for "bpf_testmod_ops" along with the +// byte offset of the "data" member inside the value type. +// +// The inner struct name is derived by trimming the "bpf_struct_ops_" prefix +// from the value's name. +func findInnerStruct(vType *btf.Struct) (*btf.Struct, uint32, error) { + innerName := strings.TrimPrefix(vType.Name, structOpsValuePrefix) + + for _, m := range vType.Members { + if st, ok := btf.As[*btf.Struct](m.Type); ok && st.Name == innerName { + return st, m.Offset.Bytes(), nil + } + } + + return nil, 0, fmt.Errorf("inner struct %q not found in %s", innerName, vType.Name) +} + +// translateStructData fills in all fields in `from` that exist in `vType` with contents of fromData. +func translateStructData(fromData []byte, vType *btf.Struct) ([]byte, error) { + vTypeData := make([]byte, int(vType.Size)) + + inner, innerOff, err := findInnerStruct(vType) + if err != nil { + return nil, err + } + + if len(fromData) < int(inner.Size) { + return nil, fmt.Errorf("inner data too short: have %d, need %d", len(fromData), inner.Size) + } + + for _, m := range inner.Members { + if m.BitfieldSize > 0 { + return nil, fmt.Errorf("bitfield %s not supported in from: %s", m.Name, inner.Name) + } + + if _, isPtr := btf.As[*btf.Pointer](m.Type); isPtr { + continue // skip func ptr + } + + kernSz, err := btf.Sizeof(m.Type) + if err != nil { + return nil, fmt.Errorf("failed vType resolve size of %s: %w", m.Name, err) + } + + srcOff := int(m.Offset.Bytes()) + dstOff := int(innerOff + m.Offset.Bytes()) + + if srcOff < 0 || srcOff+kernSz > len(fromData) { + return nil, fmt.Errorf("member %q: userdata is too small", m.Name) + } + + if dstOff < 0 || dstOff+kernSz > len(vTypeData) { + return nil, fmt.Errorf("member %q: value type is too small", m.Name) + } + + copy(vTypeData[dstOff:dstOff+kernSz], fromData[srcOff:srcOff+kernSz]) + } + + return vTypeData, nil +} + +// populateStructOpsPrograms writes progFD into `data` at the offset +func populateStructOpsPrograms(vType *btf.Struct, data []byte, programs map[string]*Program) error { + inner, innerOff, err := findInnerStruct(vType) + if err != nil { + return err + } + + for _, m := range inner.Members { + if _, isPtr := btf.As[*btf.Pointer](m.Type); !isPtr { + continue + } + + p, ok := programs[m.Name] + if !ok || p == nil { + continue + } + + dstOff := int(innerOff + m.Offset.Bytes()) + + if dstOff < 0 || dstOff+8 > len(data) { + return fmt.Errorf("member %q: kern_vdata is too small", m.Name) + } + binary.LittleEndian.PutUint64(data[dstOff:dstOff+8], uint64(p.FD())) + } + + return nil +} + // resolveKconfig resolves all variables declared in .kconfig and populates // m.Contents. Does nothing if the given m.Contents is non-empty. func resolveKconfig(m *MapSpec) error { diff --git a/collection_test.go b/collection_test.go index bcca27560..48d2172e6 100644 --- a/collection_test.go +++ b/collection_test.go @@ -14,6 +14,7 @@ import ( "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/btf" "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/sys" "github.com/cilium/ebpf/internal/testutils" "github.com/cilium/ebpf/internal/testutils/testmain" ) @@ -768,3 +769,48 @@ func ExampleCollectionSpec_LoadAndAssign() { defer objs.Program.Close() defer objs.Map.Close() } + +func TestStructOpsMapSpecSimpleLoadAndAssign(t *testing.T) { + requireTestmodOps(t) + + spec := &CollectionSpec{ + Programs: map[string]*ProgramSpec{ + "test_1": { + Name: "test_1", + Type: StructOps, + AttachTo: "bpf_testmod_ops:test_1", + License: "GPL", + Instructions: asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + }, + }, + }, + Maps: map[string]*MapSpec{ + "testmod_ops": { + Name: "testmod_ops", + Type: StructOpsMap, + Flags: sys.BPF_F_LINK, + KeySize: 4, + ValueSize: 448, + MaxEntries: 1, + Value: &btf.Struct{Name: "bpf_struct_ops_bpf_testmod_ops"}, + Contents: []MapKV{ + { + Key: uint32(0), + Value: make([]byte, 448), + }, + }, + }, + }, + } + + coll := mustNewCollection(t, spec, nil) + for name := range spec.Maps { + qt.Assert(t, qt.IsNotNil(coll.Maps[name])) + } + + for name := range spec.Programs { + qt.Assert(t, qt.IsNotNil(coll.Programs[name])) + } +} diff --git a/helpers_test.go b/helpers_test.go index 30ea66000..0b7c35989 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -30,6 +30,28 @@ var haveTestmod = sync.OnceValues(func() (bool, error) { return testmod != nil, nil }) +var haveTestmodOps = sync.OnceValues(func() (bool, error) { + haveTestMod, err := haveTestmod() + if err != nil { + return false, err + } + if !haveTestMod { + return false, nil + } + + target := btf.Type((*btf.Struct)(nil)) + _, module, err := findTargetInKernel("bpf_struct_ops_bpf_testmod_ops", &target, btf.NewCache()) + if err != nil && !errors.Is(err, btf.ErrNotFound) { + return false, err + } + if errors.Is(err, btf.ErrNotFound) { + return false, nil + } + defer module.Close() + + return true, nil +}) + func requireTestmod(tb testing.TB) { tb.Helper() @@ -44,6 +66,19 @@ func requireTestmod(tb testing.TB) { } } +func requireTestmodOps(tb testing.TB) { + tb.Helper() + + testutils.SkipOnOldKernel(tb, "5.11", "bpf_testmod") + testmodOps, err := haveTestmodOps() + if err != nil { + tb.Fatal(err) + } + if !testmodOps { + tb.Skip("bpf_testmod_ops not loaded") + } +} + func newMap(tb testing.TB, spec *MapSpec, opts *MapOptions) (*Map, error) { tb.Helper() diff --git a/map.go b/map.go index 384d71f37..f3fe5404d 100644 --- a/map.go +++ b/map.go @@ -334,7 +334,7 @@ func NewMap(spec *MapSpec) (*Map, error) { // // May return an error wrapping ErrMapIncompatible. func NewMapWithOptions(spec *MapSpec, opts MapOptions) (*Map, error) { - m, err := newMapWithOptions(spec, opts) + m, err := newMapWithOptions(spec, opts, btf.NewCache()) if err != nil { return nil, fmt.Errorf("creating map: %w", err) } @@ -347,7 +347,7 @@ func NewMapWithOptions(spec *MapSpec, opts MapOptions) (*Map, error) { return m, nil } -func newMapWithOptions(spec *MapSpec, opts MapOptions) (_ *Map, err error) { +func newMapWithOptions(spec *MapSpec, opts MapOptions, c *btf.Cache) (_ *Map, err error) { closeOnError := func(c io.Closer) { if err != nil { c.Close() @@ -397,7 +397,7 @@ func newMapWithOptions(spec *MapSpec, opts MapOptions) (_ *Map, err error) { return nil, errors.New("inner maps cannot be pinned") } - template, err := spec.InnerMap.createMap(nil) + template, err := spec.InnerMap.createMap(nil, c) if err != nil { return nil, fmt.Errorf("inner map: %w", err) } @@ -409,7 +409,7 @@ func newMapWithOptions(spec *MapSpec, opts MapOptions) (_ *Map, err error) { innerFd = template.fd } - m, err := spec.createMap(innerFd) + m, err := spec.createMap(innerFd, c) if err != nil { return nil, err } @@ -504,7 +504,7 @@ func (m *Map) memorySize() (int, error) { // createMap validates the spec's properties and creates the map in the kernel // using the given opts. It does not populate or freeze the map. -func (spec *MapSpec) createMap(inner *sys.FD) (_ *Map, err error) { +func (spec *MapSpec) createMap(inner *sys.FD, c *btf.Cache) (_ *Map, err error) { closeOnError := func(closer io.Closer) { if err != nil { closer.Close() @@ -551,14 +551,59 @@ func (spec *MapSpec) createMap(inner *sys.FD) (_ *Map, err error) { if err != nil && !errors.Is(err, btf.ErrNotSupported) { return nil, fmt.Errorf("load BTF: %w", err) } + defer handle.Close() - if handle != nil { - defer handle.Close() + if spec.Type == StructOpsMap { + if spec.Value == nil { + return nil, fmt.Errorf("struct_ops map: missing value type information") + } + + valueType, ok := btf.As[*btf.Struct](spec.Value) + if !ok { + return nil, fmt.Errorf("value must be Struct type") + } + + if spec.KeySize != 4 { + return nil, fmt.Errorf("struct_ops: KeySize must be 4") + } + + // struct_ops: resolve value type ("bpf_struct_ops_") and + // record kernel-specific BTF IDs / FDs needed for map creation. + target := btf.Type((*btf.Struct)(nil)) + s, module, err := findTargetInKernel(valueType.Name, &target, c) + if err != nil { + return nil, fmt.Errorf("lookup value type %q: %w", valueType.Name, err) + } + defer module.Close() + + vType := target.(*btf.Struct) - // Use BTF k/v during map creation. + btfValueTypeId, err := s.TypeID(vType) + if err != nil { + return nil, fmt.Errorf("lookup type_id: %w", err) + } + + attr.ValueSize = spec.ValueSize + attr.BtfVmlinuxValueTypeId = btfValueTypeId + + if handle == nil { + return nil, fmt.Errorf("struct_ops: BTF handle is not resolved") + } attr.BtfFd = uint32(handle.FD()) - attr.BtfKeyTypeId = keyTypeID - attr.BtfValueTypeId = valueTypeID + + if module != nil { + // BPF_F_VTYPE_BTF_OBJ_FD is required if the type comes from a module + attr.MapFlags |= sys.BPF_F_VTYPE_BTF_OBJ_FD + // set FD for the kernel module + attr.ValueTypeBtfObjFd = int32(module.FD()) + } + } else { + if handle != nil { + // Use BTF k/v during map creation. + attr.BtfFd = uint32(handle.FD()) + attr.BtfKeyTypeId = keyTypeID + attr.BtfValueTypeId = valueTypeID + } } } diff --git a/prog.go b/prog.go index 18a44e59f..d5cae7731 100644 --- a/prog.go +++ b/prog.go @@ -8,6 +8,8 @@ import ( "math" "path/filepath" "runtime" + "slices" + "strings" "time" "github.com/cilium/ebpf/asm" @@ -381,15 +383,48 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache) attr.AttachBtfObjFd = uint32(spec.AttachTarget.FD()) defer runtime.KeepAlive(spec.AttachTarget) } else if spec.AttachTo != "" { - module, targetID, err := findProgramTargetInKernel(spec.AttachTo, spec.Type, spec.AttachType, c) + var targetMember string + attachTo := spec.AttachTo + + if spec.Type == StructOps { + attachTo, targetMember, _ = strings.Cut(attachTo, ":") + } + + module, targetID, err := findProgramTargetInKernel(attachTo, spec.Type, spec.AttachType, c) if err != nil && !errors.Is(err, errUnrecognizedAttachType) { // We ignore errUnrecognizedAttachType since AttachTo may be non-empty // for programs that don't attach anywhere. return nil, fmt.Errorf("attach %s/%s: %w", spec.Type, spec.AttachType, err) } + if spec.Type == StructOps && targetMember != "" { + var s *btf.Spec + + target := btf.Type((*btf.Struct)(nil)) + s, module, err = findTargetInKernel(attachTo, &target, c) + if err != nil { + return nil, fmt.Errorf("lookup struct_ops kern type %q: %w", attachTo, err) + } + kType := target.(*btf.Struct) + + targetID, err = s.TypeID(kType) + if err != nil { + return nil, fmt.Errorf("type id for %s: %w", kType.TypeName(), err) + } + + idx := slices.IndexFunc(kType.Members, func(m btf.Member) bool { + return m.Name == targetMember + }) + if idx < 0 { + return nil, fmt.Errorf("member %q not found in %s", targetMember, kType.Name) + } + + // ExpectedAttachType: index of the target member in the struct + attr.ExpectedAttachType = sys.AttachType(idx) + } + attr.AttachBtfId = targetID - if module != nil { + if module != nil && attr.AttachBtfObjFd == 0 { attr.AttachBtfObjFd = uint32(module.FD()) defer module.Close() } @@ -1039,6 +1074,10 @@ func findProgramTargetInKernel(name string, progType ProgramType, attachType Att ) switch (match{progType, attachType}) { + case match{StructOps, AttachStructOps}: + typeName = name + featureName = "struct_ops " + name + target = (*btf.Struct)(nil) case match{LSM, AttachLSMMac}: typeName = "bpf_lsm_" + name featureName = name + " LSM hook" diff --git a/struct_ops.go b/struct_ops.go new file mode 100644 index 000000000..7cd4e21e9 --- /dev/null +++ b/struct_ops.go @@ -0,0 +1,3 @@ +package ebpf + +const structOpsValuePrefix = "bpf_struct_ops_" diff --git a/struct_ops_test.go b/struct_ops_test.go new file mode 100644 index 000000000..c67b5ffdf --- /dev/null +++ b/struct_ops_test.go @@ -0,0 +1,36 @@ +package ebpf + +import ( + "testing" + + "github.com/cilium/ebpf/btf" + "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/testutils" +) + +func TestCreateStructOpsMapSpecSimple(t *testing.T) { + requireTestmodOps(t) + + ms := &MapSpec{ + Name: "testmod_ops", + Type: StructOpsMap, + Flags: sys.BPF_F_LINK, + KeySize: 4, + ValueSize: 448, + MaxEntries: 1, + Value: &btf.Struct{Name: "bpf_struct_ops_bpf_testmod_ops"}, + Contents: []MapKV{ + { + Key: uint32(0), + Value: make([]byte, 448), + }, + }, + } + + m, err := NewMap(ms) + testutils.SkipIfNotSupported(t, err) + if err != nil { + t.Fatalf("creating struct_ops map failed: %v", err) + } + t.Cleanup(func() { _ = m.Close() }) +} diff --git a/types.go b/types.go index 56e318208..caedcd6c5 100644 --- a/types.go +++ b/types.go @@ -144,7 +144,7 @@ func (mt MapType) hasPerCPUValue() bool { // canStoreMapOrProgram returns true if the Map stores references to another Map // or Program. func (mt MapType) canStoreMapOrProgram() bool { - return mt.canStoreMap() || mt.canStoreProgram() + return mt.canStoreMap() || mt.canStoreProgram() || mt == StructOpsMap } // canStoreMap returns true if the map type accepts a map fd