perf: netfilter high cpu loading (#12)

* perf: netfilter high cpu loading

* support centos8 4.18.0 btf

Author: chengjoey

ebpf/plugins/netfilter/main.c

@@ -55,8 +55,11 @@ static __always_inline bool fill_event_info(struct event_t *event, struct sk_buf
 SEC("kprobe/nf_nat_setup_info")
 int kprobe_nf_nat_setup_info(struct pt_regs *ctx) {
     u64 pid_tgid;
-    struct nf_conn *conn = (struct nf_conn *)PT_REGS_PARM1(ctx);
     pid_tgid = bpf_get_current_pid_tgid();
+    if (pid_tgid == 0) {
+        return 0;
+    }
+    struct nf_conn *conn = (struct nf_conn *)PT_REGS_PARM1(ctx);
     struct nf_conn_info_t args = {
         .conn_ptr = (u64)conn,
     };
@@ -68,6 +71,9 @@ SEC("kretprobe/nf_nat_setup_info")
 int kretprobe_nf_nat_setup_info(uint ret) {
     u64 pid_tgid;
     pid_tgid = bpf_get_current_pid_tgid();
+    if (pid_tgid == 0) {
+        return 0;
+    }
     struct nf_conn_info_t *args = bpf_map_lookup_elem(&conn_maps, &pid_tgid);
     if (args == NULL) {
         return 0;

pkg/btfs/archives/4.18.0-147.8.1.el8_1.x86_64.btf

@@ -23,6 +23,7 @@ var (
 		"5.4.278-1.el7.elrepo.x86_64": "5.4.28-200.el7.x86_64.btf",
 		"5.5.5-1.el7.elrepo.x86_64":   "5.4.28-200.el7.x86_64.btf",
 		"4.18.0-147.5.1.el8_1.x86_64": "4.18.0-147.5.1.el8_1.x86_64.btf",
+		"4.18.0-147.8.1.el8_1.x86_64": "4.18.0-147.8.1.el8_1.x86_64.btf",
 	}
 )
 

pkg/btfs/btfs.go

@@ -17,7 +17,7 @@ func RunEbpf() *NetfilterObjects {
 	var bpfObj NetfilterObjects
 	if err := spec.LoadAndAssign(&bpfObj, &ebpf.CollectionOptions{
 		Programs: ebpf.ProgramOptions{
-			LogSize:     ebpf.DefaultVerifierLogSize * 10,
+			LogSize:     ebpf.DefaultVerifierLogSize,
 			KernelTypes: btfs.BtfSpec,
 		},
 	}); err != nil {
@@ -59,8 +59,8 @@ type netfilterSpecs struct {
 }
 
 type netfilterProgramSpecs struct {
-	K_iptDoTable *ebpf.ProgramSpec `ebpf:"kprobe_ipt_do_table"`
-	KrIptDoTable *ebpf.ProgramSpec `ebpf:"kretprobe_ipt_do_table"`
+	//K_iptDoTable *ebpf.ProgramSpec `ebpf:"kprobe_ipt_do_table"`
+	//KrIptDoTable *ebpf.ProgramSpec `ebpf:"kretprobe_ipt_do_table"`
 	//K_ipForward     *ebpf.ProgramSpec `ebpf:"kprobe_ip_forward"`
 	//Kr_ipForward    *ebpf.ProgramSpec `ebpf:"kretprobe_ip_forward"`
 	K_natSetUpInfo  *ebpf.ProgramSpec `ebpf:"kprobe_nf_nat_setup_info"`
@@ -69,10 +69,10 @@ type netfilterProgramSpecs struct {
 
 type netfilterMapSpecs struct {
 	EventBuf *ebpf.MapSpec `ebpf:"event_buf"`
-	IptMaps  *ebpf.MapSpec `ebpf:"ipt_maps"`
+	//IptMaps  *ebpf.MapSpec `ebpf:"ipt_maps"`
 	//IpRcvMaps  *ebpf.MapSpec `ebpf:"ip_rcv_maps"`
-	NfConnMaps *ebpf.MapSpec `ebpf:"conn_maps"`
-	NfConnBuf  *ebpf.MapSpec `ebpf:"nf_conn_maps"`
+	//NfConnMaps *ebpf.MapSpec `ebpf:"conn_maps"`
+	//NfConnBuf  *ebpf.MapSpec `ebpf:"nf_conn_maps"`
 }
 
 type NetfilterObjects struct {
@@ -88,26 +88,26 @@ func (o *NetfilterObjects) Close() error {
 }
 
 type netfilterMaps struct {
-	EventBuf *ebpf.Map `ebpf:"event_buf"`
-	IptMaps  *ebpf.Map `ebpf:"ipt_maps"`
+	//EventBuf *ebpf.Map `ebpf:"event_buf"`
+	//IptMaps  *ebpf.Map `ebpf:"ipt_maps"`
 	//IpRcvMaps  *ebpf.Map `ebpf:"ip_rcv_maps"`
-	NfConnMaps *ebpf.Map `ebpf:"conn_maps"`
-	NfConnBuf  *ebpf.Map `ebpf:"nf_conn_maps"`
+	//NfConnMaps *ebpf.Map `ebpf:"conn_maps"`
+	NfConnBuf *ebpf.Map `ebpf:"nf_conn_maps"`
 }
 
 func (m *netfilterMaps) Close() error {
 	return _NetfilterClose(
-		m.EventBuf,
-		m.IptMaps,
+		//m.EventBuf,
+		//m.IptMaps,
 		//m.IpRcvMaps,
-		m.NfConnMaps,
+		//m.NfConnMaps,
 		m.NfConnBuf,
 	)
 }
 
 type netfilterPrograms struct {
-	K_iptDoTable *ebpf.Program `ebpf:"kprobe_ipt_do_table"`
-	KrIptDoTable *ebpf.Program `ebpf:"kretprobe_ipt_do_table"`
+	//K_iptDoTable *ebpf.Program `ebpf:"kprobe_ipt_do_table"`
+	//KrIptDoTable *ebpf.Program `ebpf:"kretprobe_ipt_do_table"`
 	//K_ipFroward     *ebpf.Program `ebpf:"kprobe_ip_forward"`
 	//Kr_ipFroward    *ebpf.Program `ebpf:"kretprobe_ip_forward""`
 	K_natSetUpInfo  *ebpf.Program `ebpf:"kprobe_nf_nat_setup_info"`
@@ -116,8 +116,8 @@ type netfilterPrograms struct {
 
 func (p *netfilterPrograms) Close() error {
 	return _NetfilterClose(
-		p.K_iptDoTable,
-		p.KrIptDoTable,
+		//p.K_iptDoTable,
+		//p.KrIptDoTable,
 		//p.K_ipFroward,
 		//p.Kr_ipFroward,
 		p.K_natSetUpInfo,

pkg/plugins/netfilter/ebpf/bpf_netfilter.go

@@ -58,12 +58,13 @@ func (p *provider) Gather(c chan *metric.Metric) {
 	}
 	defer krpNat.Close()
 
+	connMap := obj.NfConnBuf
+	var (
+		key uint64
+		val []byte
+	)
 	for {
-		var (
-			key uint64
-			val []byte
-		)
-		for obj.NfConnBuf.Iterate().Next(&key, &val) {
+		for connMap.Iterate().Next(&key, &val) {
 			if err := obj.NfConnBuf.Delete(key); err != nil {
 				panic(err)
 			}
@@ -83,6 +84,7 @@ func (p *provider) Gather(c chan *metric.Metric) {
 			}
 			p.natCache.Set(fmt.Sprintf("%s:%d", srcIP, event.OriSport), natInfo, time.Minute)
 		}
+		time.Sleep(100 * time.Millisecond)
 	}
 }
 

pkg/plugins/netfilter/netfilter.go

@@ -77,7 +77,7 @@ func (p *provider) Convert(m *ebpf.Metric) *metric.Metric {
 
 	sourcePod, err := p.kprobeHelper.GetPodByUID(m.SourceIP)
 	if err != nil {
-		p.l.Errorf("failed to get pod by uid: %s, err: %v", m.SourceIP, err)
+		p.l.Debugf("failed to get pod by uid: %s, err: %v", m.SourceIP, err)
 	} else {
 		// source platform metadata
 		output.OrgName = sourcePod.Labels["DICE_ORG_NAME"]