add container_nodejs_event_loop_blocked_time_seconds_total

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23.8-bullseye AS builder
+FROM golang:1.24.7-trixie AS builder
 RUN apt update && apt install -y libsystemd-dev
 WORKDIR /tmp/src
 COPY go.mod .

containers/container.go

@@ -120,7 +120,6 @@ type Container struct {
 
 	delays      Delays
 	delaysByPid map[uint32]Delays
-	delaysLock  sync.Mutex
 
 	listens map[netaddr.IPPort]map[uint32]*ListenDetails
 
@@ -137,6 +136,7 @@ type Container struct {
 
 	oomKills                 int
 	pythonThreadLockWaitTime time.Duration
+	nodejsStats              *ebpftracer.NodejsStats
 
 	mounts     map[string]proc.MountInfo
 	seenMounts map[uint64]struct{}
@@ -147,7 +147,7 @@ type Container struct {
 
 	registry *Registry
 
-	lock sync.RWMutex
+	lock sync.Mutex
 
 	done chan struct{}
 }
@@ -232,10 +232,10 @@ func (c *Container) Describe(ch chan<- *prometheus.Desc) {
 }
 
 func (c *Container) Collect(ch chan<- prometheus.Metric) {
-	c.registry.updateTrafficStatsIfNecessary()
+	c.registry.updateStatsFromEbpfMapsIfNecessary()
 
-	c.lock.RLock()
-	defer c.lock.RUnlock()
+	c.lock.Lock()
+	defer c.lock.Unlock()
 
 	if c.metadata.image != "" || c.metadata.systemdTriggeredBy != "" {
 		ch <- gauge(metrics.ContainerInfo, 1, c.metadata.image, c.metadata.systemdTriggeredBy)
@@ -402,6 +402,9 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 	if c.pythonThreadLockWaitTime > 0 {
 		ch <- counter(metrics.PythonThreadLockWaitTime, c.pythonThreadLockWaitTime.Seconds())
 	}
+	if c.nodejsStats != nil {
+		ch <- counter(metrics.NodejsEventLoopBlockedTime, c.nodejsStats.EventLoopBlockedTime.Seconds())
+	}
 
 	if c.dnsStats.Requests != nil {
 		c.dnsStats.Requests.Collect(ch)
@@ -816,8 +819,6 @@ func (c *Container) onRetransmission(src netaddr.IPPort, dst netaddr.IPPort) boo
 }
 
 func (c *Container) updateDelays() {
-	c.delaysLock.Lock()
-	defer c.delaysLock.Unlock()
 	for pid := range c.processes {
 		stats, err := TaskstatsTGID(pid)
 		if err != nil {
@@ -832,6 +833,23 @@ func (c *Container) updateDelays() {
 	}
 }
 
+func (c *Container) updateNodejsStats(s NodejsStatsUpdate) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	p := c.processes[s.Pid]
+	if p == nil || p.nodejsPrevStats == nil {
+		return
+	}
+	if delta := s.Stats.EventLoopBlockedTime - p.nodejsPrevStats.EventLoopBlockedTime; delta > 0 {
+		if c.nodejsStats == nil {
+			c.nodejsStats = &ebpftracer.NodejsStats{}
+		}
+		c.nodejsStats.EventLoopBlockedTime += delta
+	}
+	p.nodejsPrevStats = &s.Stats
+}
+
 func (c *Container) getMounts() map[string]map[string]*proc.FSStat {
 	if len(c.mounts) == 0 {
 		return nil

containers/metrics.go

@@ -49,7 +49,8 @@ var metrics = struct {
 	JvmSafepointTime     *prometheus.Desc
 	JvmSafepointSyncTime *prometheus.Desc
 
-	PythonThreadLockWaitTime *prometheus.Desc
+	PythonThreadLockWaitTime   *prometheus.Desc
+	NodejsEventLoopBlockedTime *prometheus.Desc
 
 	GpuUsagePercent       *prometheus.Desc
 	GpuMemoryUsagePercent *prometheus.Desc
@@ -102,7 +103,8 @@ var metrics = struct {
 
 	Ip2Fqdn: metric("ip_to_fqdn", "Mapping IP addresses to FQDNs based on DNS requests initiated by containers", "ip", "fqdn"),
 
-	PythonThreadLockWaitTime: metric("container_python_thread_lock_wait_time_seconds", "Time spent waiting acquiring GIL in seconds"),
+	PythonThreadLockWaitTime:   metric("container_python_thread_lock_wait_time_seconds", "Time spent waiting acquiring GIL in seconds"),
+	NodejsEventLoopBlockedTime: metric("container_nodejs_event_loop_blocked_time_seconds_total", "Total time the Node.js event loop spent blocked"),
 
 	GpuUsagePercent:       metric("container_resources_gpu_usage_percent", "Percent of GPU compute resources used by the container", "gpu_uuid"),
 	GpuMemoryUsagePercent: metric("container_resources_gpu_memory_usage_percent", "Percent of GPU memory used by the container", "gpu_uuid"),

containers/process.go

@@ -42,6 +42,8 @@ type Process struct {
 	goTlsUprobesChecked   bool
 	openSslUprobesChecked bool
 	pythonGilChecked      bool
+	nodejsChecked         bool
+	nodejsPrevStats       *ebpftracer.NodejsStats
 
 	gpuUsageSamples []gpu.ProcessUsageSample
 }
@@ -84,6 +86,7 @@ func (p *Process) instrument(tracer *ebpftracer.Tracer) {
 			cmdline := proc.GetCmdline(p.Pid)
 			if dest != "/" && len(cmdline) > 0 {
 				p.instrumentPython(cmdline, tracer)
+				p.instrumentNodejs(dest, tracer)
 				if dotNetAppName, err := dotNetApp(cmdline, p.Pid); err == nil {
 					if dotNetAppName != "" {
 						p.dotNetMonitor = NewDotNetMonitor(p.ctx, p.Pid, dotNetAppName)
@@ -113,6 +116,18 @@ func (p *Process) instrumentPython(cmdline []byte, tracer *ebpftracer.Tracer) {
 	p.uprobes = append(p.uprobes, tracer.AttachPythonThreadLockProbes(p.Pid)...)
 }
 
+func (p *Process) instrumentNodejs(exe string, tracer *ebpftracer.Tracer) {
+	if p.nodejsChecked {
+		return
+	}
+	p.nodejsChecked = true
+	if !nodejsCmd.MatchString(exe) {
+		return
+	}
+	p.nodejsPrevStats = &ebpftracer.NodejsStats{}
+	p.uprobes = append(p.uprobes, tracer.AttachNodejsProbes(p.Pid, exe)...)
+}
+
 func (p *Process) addGpuUsageSample(sample gpu.ProcessUsageSample) {
 	p.removeOldGpuUsageSamples(sample.Timestamp.Add(-gpuStatsWindow))
 	p.gpuUsageSamples = append(p.gpuUsageSamples, sample)

containers/registry.go

@@ -58,9 +58,10 @@ type Registry struct {
 
 	processInfoCh chan<- ProcessInfo
 
-	trafficStatsLastUpdated time.Time
-	trafficStatsLock        sync.Mutex
-	trafficStatsUpdateCh    chan *TrafficStatsUpdate
+	ebpfStatsLastUpdated time.Time
+	ebpfStatsLock        sync.Mutex
+	trafficStatsUpdateCh chan *TrafficStatsUpdate
+	nodejsStatsUpdateCh  chan *NodejsStatsUpdate
 
 	gpuProcessUsageSampleChan chan gpu.ProcessUsageSample
 }
@@ -117,6 +118,7 @@ func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, gp
 		tracer: ebpftracer.NewTracer(hostNetNs, selfNetNs, *flags.DisableL7Tracing),
 
 		trafficStatsUpdateCh: make(chan *TrafficStatsUpdate),
+		nodejsStatsUpdateCh:  make(chan *NodejsStatsUpdate),
 
 		gpuProcessUsageSampleChan: gpuProcessUsageSampleChan,
 	}
@@ -211,6 +213,13 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 			if c := r.containersByPid[u.Pid]; c != nil {
 				c.updateTrafficStats(u)
 			}
+		case u := <-r.nodejsStatsUpdateCh:
+			if u == nil {
+				continue
+			}
+			if c := r.containersByPid[u.Pid]; c != nil {
+				c.updateNodejsStats(*u)
+			}
 		case sample := <-r.gpuProcessUsageSampleChan:
 			if c := r.containersByPid[sample.Pid]; c != nil {
 				if p := c.processes[sample.Pid]; p != nil {
@@ -391,13 +400,21 @@ func (r *Registry) getOrCreateContainer(pid uint32) *Container {
 	return c
 }
 
-func (r *Registry) updateTrafficStatsIfNecessary() {
-	r.trafficStatsLock.Lock()
-	defer r.trafficStatsLock.Unlock()
+func (r *Registry) updateStatsFromEbpfMapsIfNecessary() {
+	r.ebpfStatsLock.Lock()
+	defer r.ebpfStatsLock.Unlock()
 
-	if time.Now().Sub(r.trafficStatsLastUpdated) < MinTrafficStatsUpdateInterval {
+	if time.Now().Sub(r.ebpfStatsLastUpdated) < MinTrafficStatsUpdateInterval {
 		return
 	}
+
+	r.updateTrafficStats()
+	r.updateNodejsStats()
+
+	r.ebpfStatsLastUpdated = time.Now()
+}
+
+func (r *Registry) updateTrafficStats() {
 	iter := r.tracer.ActiveConnectionsIterator()
 	cid := ebpftracer.ConnectionId{}
 	stats := ebpftracer.Connection{}
@@ -413,7 +430,21 @@ func (r *Registry) updateTrafficStatsIfNecessary() {
 		klog.Warningln(err)
 	}
 	r.trafficStatsUpdateCh <- nil
-	r.trafficStatsLastUpdated = time.Now()
+}
+
+func (r *Registry) updateNodejsStats() {
+	iter := r.tracer.NodejsStatsIterator()
+	var pid uint64
+	stats := ebpftracer.NodejsStats{}
+
+	for iter.Next(&pid, &stats) {
+		r.nodejsStatsUpdateCh <- &NodejsStatsUpdate{Pid: uint32(pid), Stats: stats}
+	}
+
+	if err := iter.Err(); err != nil {
+		klog.Warningln(err)
+	}
+	r.nodejsStatsUpdateCh <- nil
 }
 
 func (r *Registry) getDomain(ip netaddr.IP) *common.Domain {
@@ -527,3 +558,8 @@ type TrafficStatsUpdate struct {
 	BytesSent     uint64
 	BytesReceived uint64
 }
+
+type NodejsStatsUpdate struct {
+	Pid   uint32
+	Stats ebpftracer.NodejsStats
+}

ebpftracer/ebpf/ebpf.c

@@ -39,6 +39,7 @@ struct trace_event_raw_sys_exit__stub {
 	long int ret;
 };
 
+#include "nodejs.c"
 #include "proc.c"
 #include "file.c"
 #include "tcp/conntrack.c"

ebpftracer/ebpf/nodejs.c

@@ -0,0 +1,100 @@
+struct nodejs_proc_stats {
+    __u64 event_loop_blocked_time;
+};
+
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(key_size, sizeof(__u64));
+    __uint(value_size, sizeof(__u64));
+    __uint(max_entries, 10240);
+} nodejs_prev_event_loop_iter SEC(".maps");
+
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(key_size, sizeof(__u64));
+    __uint(value_size, sizeof(__u64));
+    __uint(max_entries, 10240);
+} nodejs_current_io_cb SEC(".maps");
+
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(key_size, sizeof(__u64));
+    __uint(value_size, sizeof(struct nodejs_proc_stats));
+    __uint(max_entries, 10240);
+} nodejs_stats SEC(".maps");
+
+SEC("uprobe/uv_io_poll_exit")
+int uv_io_poll_exit(struct pt_regs *ctx) {
+    __u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+    __u64 timestamp = bpf_ktime_get_ns();
+    if ((__u32)pid_tgid != (__u32)pid) {
+        return 0;
+    }
+    bpf_map_update_elem(&nodejs_prev_event_loop_iter, &pid, &timestamp, BPF_ANY);
+    return 0;
+}
+
+SEC("uprobe/uv_io_poll_enter")
+int uv_io_poll_enter(struct pt_regs *ctx) {
+    __u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+    if ((__u32)pid_tgid != (__u32)pid) {
+        return 0;
+    }
+    __u64 *prev = bpf_map_lookup_elem(&nodejs_prev_event_loop_iter, &pid);
+    if (!prev) {
+        return 0;
+    }
+    __u64 duration = bpf_ktime_get_ns() - *prev;
+    bpf_map_delete_elem(&nodejs_prev_event_loop_iter, &pid);
+    struct nodejs_proc_stats *stats = bpf_map_lookup_elem(&nodejs_stats, &pid);
+    if (!stats) {
+        struct nodejs_proc_stats s = {};
+        bpf_map_update_elem(&nodejs_stats, &pid, &s, BPF_ANY);
+        stats = bpf_map_lookup_elem(&nodejs_stats, &pid);
+        if (!stats) {
+            return 0;
+        }
+    }
+    __sync_fetch_and_add(&stats->event_loop_blocked_time, duration);
+    return 0;
+}
+
+SEC("uprobe/uv_io_cb_enter")
+int uv_io_cb_enter(struct pt_regs *ctx) {
+    __u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+    if ((__u32)pid_tgid != (__u32)pid) {
+        return 0;
+    }
+    __u64 timestamp = bpf_ktime_get_ns();
+    bpf_map_update_elem(&nodejs_current_io_cb, &pid, &timestamp, BPF_ANY);
+    return 0;
+}
+
+SEC("uprobe/uv_io_cb_exit")
+int uv_io_cb_exit(struct pt_regs *ctx) {
+    __u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+    if ((__u32)pid_tgid != (__u32)pid) {
+        return 0;
+    }
+    __u64 *start = bpf_map_lookup_elem(&nodejs_current_io_cb, &pid);
+    if (!start) {
+        return 0;
+    }
+    __u64 duration = bpf_ktime_get_ns() - *start;
+    bpf_map_delete_elem(&nodejs_current_io_cb, &pid);
+    struct nodejs_proc_stats *stats = bpf_map_lookup_elem(&nodejs_stats, &pid);
+    if (!stats) {
+        struct nodejs_proc_stats s = {};
+        bpf_map_update_elem(&nodejs_stats, &pid, &s, BPF_ANY);
+        stats = bpf_map_lookup_elem(&nodejs_stats, &pid);
+        if (!stats) {
+            return 0;
+        }
+    }
+    __sync_fetch_and_add(&stats->event_loop_blocked_time, duration);
+    return 0;
+}

ebpftracer/ebpf/proc.c

@@ -57,9 +57,15 @@ SEC("tracepoint/sched/sched_process_exit")
 int sched_process_exit(struct trace_event_raw_sched_process_template__stub *args)
 {
     __u64 id = bpf_get_current_pid_tgid();
-    if (id >> 32 != (__u32)id) { // skipping threads
+    __u64 pid = id >> 32;
+    if (pid != (__u32)id) { // skipping threads
         return 0;
     }
+
+    bpf_map_delete_elem(&nodejs_stats, &pid);
+    bpf_map_delete_elem(&nodejs_prev_event_loop_iter, &pid);
+    bpf_map_delete_elem(&nodejs_current_io_cb, &pid);
+
     struct proc_event e = {
         .type = EVENT_TYPE_PROCESS_EXIT,
         .pid = args->pid,