replace uretprobes with uprobes using return offsets to avoid segmentation faults

ebpftracer/elf.go

@@ -0,0 +1,169 @@
+package ebpftracer
+
+import (
+	"debug/elf"
+	"fmt"
+	"io"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/link"
+	"golang.org/x/arch/arm64/arm64asm"
+	"golang.org/x/arch/x86/x86asm"
+)
+
+type Symbol struct {
+	s       *elf.Symbol
+	f       *ELFFile
+	address uint64
+}
+
+func (s *Symbol) Name() string {
+	return s.s.Name
+}
+
+func (s *Symbol) Address() uint64 {
+	if s.address == 0 {
+		s.address = s.s.Value
+		for _, p := range s.f.elf.Progs {
+			if p.Type != elf.PT_LOAD || (p.Flags&elf.PF_X) == 0 {
+				continue
+			}
+			if p.Vaddr <= s.s.Value && s.s.Value < (p.Vaddr+p.Memsz) {
+				s.address = s.s.Value - p.Vaddr + p.Off
+				break
+			}
+		}
+	}
+	return s.address
+}
+
+func (s *Symbol) ReturnOffsets() ([]int, error) {
+	text, reader, err := s.f.getTextSectionAndReader()
+	if err != nil {
+		return nil, err
+	}
+
+	sStart := s.s.Value - text.Addr
+	_, err = reader.Seek(int64(sStart), io.SeekStart)
+	if err != nil {
+		return nil, err
+	}
+	sBytes := make([]byte, s.s.Size)
+	_, err = reader.Read(sBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	offsets := getReturnOffsets(s.f.elf.Machine, sBytes)
+	if len(offsets) == 0 {
+		return nil, fmt.Errorf("no offsets found")
+	}
+	return offsets, nil
+}
+
+func (s *Symbol) AttachUprobe(exe *link.Executable, prog *ebpf.Program, pid uint32) (link.Link, error) {
+	return exe.Uprobe(s.Name(), prog, &link.UprobeOptions{Address: s.Address(), PID: int(pid)})
+}
+
+func (s *Symbol) AttachUretprobes(exe *link.Executable, prog *ebpf.Program, pid uint32) ([]link.Link, error) {
+	returnOffsets, err := s.ReturnOffsets()
+	if err != nil {
+		return nil, err
+	}
+	var links []link.Link
+	for _, offset := range returnOffsets {
+		l, err := exe.Uprobe("pthread_cond_timedwait", prog, &link.UprobeOptions{Address: s.Address(), Offset: uint64(offset), PID: int(pid)})
+		if err != nil {
+			return links, err
+		}
+		links = append(links, l)
+	}
+
+	return links, nil
+}
+
+type ELFFile struct {
+	elf               *elf.File
+	symbols           []elf.Symbol
+	textSection       *elf.Section
+	textSectionReader io.ReadSeeker
+}
+
+func OpenELFFile(path string) (*ELFFile, error) {
+	file, err := elf.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	return &ELFFile{elf: file}, nil
+}
+
+func (f *ELFFile) readSymbols() error {
+	symbols, _ := f.elf.Symbols()
+	dyn, _ := f.elf.DynamicSymbols()
+
+	if len(symbols) == 0 && len(dyn) == 0 {
+		return fmt.Errorf("no symbols found")
+	}
+	f.symbols = append(symbols, dyn...)
+	return nil
+}
+
+func (f *ELFFile) GetSymbol(name string) (*Symbol, error) {
+	if f.symbols == nil {
+		if err := f.readSymbols(); err != nil {
+			return nil, err
+		}
+	}
+	var es *elf.Symbol
+	for _, s := range f.symbols {
+		if elf.ST_TYPE(s.Info) != elf.STT_FUNC || s.Size == 0 || s.Value == 0 {
+			continue
+		}
+		if s.Name == name && s.VersionIndex&0x8000 == 0 {
+			es = &s
+			break
+		}
+	}
+	if es == nil {
+		return nil, fmt.Errorf("symbol %s not found", name)
+	}
+	return &Symbol{s: es, f: f}, nil
+}
+
+func (f *ELFFile) getTextSectionAndReader() (*elf.Section, io.ReadSeeker, error) {
+	if f.textSection == nil {
+		f.textSection = f.elf.Section(".text")
+		if f.textSection == nil {
+			return nil, nil, fmt.Errorf("no .text")
+		}
+		f.textSectionReader = f.textSection.Open()
+	}
+	return f.textSection, f.textSectionReader, nil
+}
+
+func (f *ELFFile) Close() error {
+	return f.elf.Close()
+}
+
+func getReturnOffsets(machine elf.Machine, instructions []byte) []int {
+	var res []int
+	switch machine {
+	case elf.EM_X86_64:
+		for i := 0; i < len(instructions); {
+			ins, err := x86asm.Decode(instructions[i:], 64)
+			if err == nil && ins.Op == x86asm.RET {
+				res = append(res, i)
+			}
+			i += ins.Len
+		}
+	case elf.EM_AARCH64:
+		for i := 0; i < len(instructions); {
+			ins, err := arm64asm.Decode(instructions[i:])
+			if err == nil && ins.Op == arm64asm.RET {
+				res = append(res, i)
+			}
+			i += 4
+		}
+	}
+	return res
+}

ebpftracer/nodejs.go

@@ -25,55 +25,65 @@ func (t *Tracer) AttachNodejsProbes(pid uint32, exe string) []link.Link {
 		klog.InfofDepth(1, "pid=%d lib=%s: %s", pid, libPath, msg)
 	}
 
-	var (
-		lastErr error
-		links   []link.Link
-		libPath string
-	)
-	for _, libPath = range append(getLibuv(pid), proc.Path(pid, "root", exe)) {
-		exe, err := link.OpenExecutable(libPath)
-		if err != nil {
-			log(libPath, "failed to open executable", err)
-			return nil
-		}
-		options := &link.UprobeOptions{PID: int(pid)}
-		var uprobe, uretprobe link.Link
-		uprobe, lastErr = exe.Uprobe("uv__io_poll", t.uprobes["uv_io_poll_enter"], options)
-		if lastErr != nil {
-			continue
+	for _, libPath := range append(getLibuv(pid), proc.Path(pid, "root", exe)) {
+		if links, err := t.attachNodejsUprobes(libPath, pid); err == nil {
+			log(libPath, "nodejs uprobes attached", nil)
+			return links
+		} else {
+			log(libPath, "failed to attach nodejs uprobes", err)
 		}
+	}
+	return nil
+}
 
-		links = append(links, uprobe)
-		uretprobe, lastErr = exe.Uretprobe("uv__io_poll", t.uprobes["uv_io_poll_exit"], options)
-		if lastErr != nil {
-			continue
-		}
+func (t *Tracer) attachNodejsUprobes(libPath string, pid uint32) ([]link.Link, error) {
+	exe, err := link.OpenExecutable(libPath)
+	if err != nil {
+		return nil, err
+	}
+	ef, err := OpenELFFile(libPath)
+	if err != nil {
+		return nil, err
+	}
+	defer ef.Close()
+
+	s, err := ef.GetSymbol("uv__io_poll")
+	if err != nil {
+		return nil, err
+	}
+	l, err := s.AttachUprobe(exe, t.uprobes["uv_io_poll_enter"], pid)
+	if err != nil {
+		return nil, err
+	}
+	var links []link.Link
+	links = append(links, l)
 
-		links = append(links, uretprobe)
+	ls, err := s.AttachUretprobes(exe, t.uprobes["uv_io_poll_exit"], pid)
+	links = append(links, ls...)
+	if err != nil {
+		for _, l := range links {
+			_ = l.Close()
+		}
+		return nil, err
+	}
 
-		for _, cb := range []string{"uv__stream_io", "uv__async_io", "uv__poll_io", "uv__server_io", "uv__udp_io"} {
-			uprobe, lastErr = exe.Uprobe(cb, t.uprobes["uv_io_cb_enter"], options)
-			if lastErr != nil {
-				break
-			}
-			links = append(links, uprobe)
-			uretprobe, lastErr = exe.Uretprobe(cb, t.uprobes["uv_io_cb_exit"], options)
-			if lastErr != nil {
-				break
-			}
-			links = append(links, uretprobe)
+	for _, cb := range []string{"uv__stream_io", "uv__async_io", "uv__poll_io", "uv__server_io", "uv__udp_io"} {
+		s, err = ef.GetSymbol(cb)
+		if err != nil {
+			break
 		}
-		if lastErr != nil {
-			continue
+		l, err = s.AttachUprobe(exe, t.uprobes["uv_io_cb_enter"], pid)
+		if err != nil {
+			break
+		}
+		links = append(links, l)
+		ls, err = s.AttachUretprobes(exe, t.uprobes["uv_io_cb_exit"], pid)
+		links = append(links, ls...)
+		if err != nil {
+			break
 		}
-
-		log(libPath, "nodejs uprobes attached", nil)
-		break
-	}
-	if lastErr != nil {
-		log(libPath, "failed to attach uprobe", lastErr)
 	}
-	return links
+	return links, nil
 }
 
 func getLibuv(pid uint32) []string {

ebpftracer/python.go

@@ -14,7 +14,7 @@ import (
 
 var (
 	libcRegexp = regexp.MustCompile(`libc[\.-]`)
-	muslRegexp = regexp.MustCompile(`musl[\.-]`)
+	muslRegexp = regexp.MustCompile(`ld-musl[\.-]`)
 )
 
 func (t *Tracer) AttachPythonThreadLockProbes(pid uint32) []link.Link {
@@ -32,38 +32,55 @@ func (t *Tracer) AttachPythonThreadLockProbes(pid uint32) []link.Link {
 	}
 
 	var (
-		lastErr error
-		links   []link.Link
-		libPath string
+		links []link.Link
+		err   error
 	)
 
-	for _, libPath = range getPthreadLibs(pid) {
-		exe, err := link.OpenExecutable(libPath)
-		if err != nil {
-			log(libPath, "failed to open executable", err)
-			return nil
-		}
-		options := &link.UprobeOptions{PID: int(pid)}
-		var uprobe, uretprobe link.Link
-		uprobe, lastErr = exe.Uprobe("pthread_cond_timedwait", t.uprobes["pthread_cond_timedwait_enter"], options)
-		if lastErr != nil {
-			continue
-		}
-		links = append(links, uprobe)
-		uretprobe, lastErr = exe.Uretprobe("pthread_cond_timedwait", t.uprobes["pthread_cond_timedwait_exit"], options)
-		if lastErr != nil {
-			continue
+	for _, libPath := range getPthreadLibs(pid) {
+		if links, err = t.attachPythonUprobes(libPath, pid); err == nil {
+			log(libPath, "python uprobes attached", nil)
+			return links
+		} else {
+			log(libPath, "failed to attach python uprobes", err)
 		}
-		links = append(links, uretprobe)
-		log(libPath, "python uprobes attached", nil)
-		break
 	}
-	if lastErr != nil {
-		log(libPath, "failed to attach uprobe", lastErr)
+	if len(links) > 0 {
+
 	}
 	return links
 }
 
+func (t *Tracer) attachPythonUprobes(libPath string, pid uint32) ([]link.Link, error) {
+	exe, err := link.OpenExecutable(libPath)
+	if err != nil {
+		return nil, err
+	}
+	ef, err := OpenELFFile(libPath)
+	if err != nil {
+		return nil, err
+	}
+	defer ef.Close()
+
+	s, err := ef.GetSymbol("pthread_cond_timedwait")
+	if err != nil {
+		return nil, err
+	}
+	l, err := s.AttachUprobe(exe, t.uprobes["pthread_cond_timedwait_enter"], pid)
+	if err != nil {
+		return nil, err
+	}
+	links := []link.Link{l}
+	ls, err := s.AttachUretprobes(exe, t.uprobes["pthread_cond_timedwait_exit"], pid)
+	links = append(links, ls...)
+	if err != nil {
+		for _, l := range links {
+			_ = l.Close()
+		}
+		return nil, err
+	}
+	return links, nil
+}
+
 func getPthreadLibs(pid uint32) []string {
 	f, err := os.Open(proc.Path(pid, "maps"))
 	if err != nil {