tools: add numasched: trace task switch between NUMA (iovisor#4396)

Tasks frequently switch between NUMA, often resulting in poor performance.

programs running on the same NUMA will have better performance (usually at
the memory level). If processes are migrating between different CPUs and
these CPUs belong to different NUMAs, this tool provides an effective means
of tracing to optimize the program or configuration.

For example:

 Terminal 1:

  $ numactl --hardware
  available: 4 nodes (0-3)
  node 0 cpus: 0 - 23
  node 1 cpus: 24 - 47
  node 2 cpus: 48 - 71
  node 3 cpus: 72 - 95

 Terminal 2:

  $ sudo ./numasched.py
  Tracing task NUMA switch...
  TIME     PID      SRC_NID     DST_NID  COMM

 Terminal 3:

  $ taskset -c 1 yes >/dev/null &
  $ taskset -p 0x1000000 $(pidof yes)
  $ taskset -p 0x1000000000000 $(pidof yes)

 Then, Terminal 2:

  $ sudo ./numasched.py
  Tracing task NUMA switch...
  TIME     PID      TID      SRC_NID     DST_NID  COMM
  20:55:35 355842   355842   0        -> 1        b'yes'
  20:55:50 355842   355842   1        -> 2        b'yes'

Signed-off-by: Rong Tao <[email protected]>

Author: Rtoax

man/man8/numasched.8

@@ -0,0 +1,66 @@
+.TH numasched 8  "2022-12-14" "USER COMMANDS"
+.SH NAME
+numasched \- Tracing task switch NUMA. Uses bcc/eBPF.
+.SH SYNOPSIS
+.B numasched
+.SH DESCRIPTION
+numasched tracked task switch of NUMA.
+
+This program is also a basic example of bcc and tracepoint.
+
+Since this uses BPF, only the root user can use this tool.
+.SH REQUIREMENTS
+CONFIG_BPF and bcc.
+.SH OPTIONS
+.TP
+\-p, --pid PID
+Trace this PID only.
+.TP
+\-t, --tid TID
+Trace this TID only.
+.TP
+\-c, --comm COMM
+Trace this COMM only.
+.SH EXAMPLES
+.TP
+Tracing task switch NUMA:
+#
+.B numasched
+.TP
+Trace PID 181 only:
+#
+.B numasched \-p 181
+.SH FIELDS
+.TP
+TIME
+A timestamp on the output, in "HH:MM:SS" format.
+.TP
+PID
+The process ID.
+.TP
+TID
+The thread ID.
+.TP
+SRC_NID
+Source NUMA ID.
+.TP
+DST_NID
+Target NUMA ID.
+.TP
+COMM
+The process COMM.
+.SH SOURCE
+This is from bcc.
+.IP
+https://github.com/iovisor/bcc
+.PP
+Also look in the bcc distribution for a companion _examples.txt file
+containing example usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Rong Tao <[email protected]>
+.SH SEE ALSO
+opensnoop(8)

tools/numasched.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# @lint-avoid-python-3-compatibility-imports
+#
+# numasched  Trace task NUMA switch
+#            For Linux, uses BCC, eBPF.
+#
+# USAGE: numasched [-p PID] [-t TID] [-c COMM]
+#
+# This script tracks NUMA migrations of tasks, and in general, frequent
+# NUMA migrations can cause poor performance.
+#
+# Copyright 2022 CESTC, Co.
+# Licensed under the Apache License, Version 2.0 (the "License")
+#
+# 14-Dec-2022   Rong Tao    Created this.
+
+from __future__ import print_function
+from bcc import BPF
+import argparse
+from time import strftime
+from socket import inet_ntop, AF_INET, AF_INET6
+from struct import pack
+from time import sleep
+
+
+# arguments
+examples = """examples:
+    ./numasched             # trace all processes
+    ./numasched -p 185      # trace PID 185 only
+"""
+parser = argparse.ArgumentParser(
+    description="Trace task NUMA switch",
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    epilog=examples)
+parser.add_argument("-p", "--pid",
+    help="trace this PID only")
+parser.add_argument("-t", "--tid",
+    help="trace this TID only")
+parser.add_argument("-c", "--comm",
+    help="trace this COMM only")
+args = parser.parse_args()
+
+
+bpf_text = """
+#include <linux/sched.h>
+#include <linux/topology.h>
+
+struct data_t {
+    char comm[TASK_COMM_LEN];
+    u32 pid;
+    u32 tid;
+    u32 old_nid;
+    u32 new_nid;
+};
+BPF_PERF_OUTPUT(events);
+
+struct val_t {
+    u32 nid;
+};
+BPF_HASH(numaid_info, u32, struct val_t);
+
+
+TRACEPOINT_PROBE(sched, sched_switch)
+{
+    u64 pid_tgid = bpf_get_current_pid_tgid();
+    u32 pid = pid_tgid >> 32;
+    u32 tid = (u32)pid_tgid;
+    u32 new_nid = bpf_get_numa_node_id();
+    struct val_t val = {}, *valp;
+    u32 old_nid;
+
+    if (FILTER_PID)
+        return 0;
+
+    if (FILTER_TID)
+        return 0;
+
+    val.nid = new_nid;
+
+    valp = numaid_info.lookup(&tid);
+    if (!valp)
+        goto update;
+
+    old_nid = valp->nid;
+
+    if (old_nid != new_nid) {
+        struct data_t data = {};
+
+        bpf_get_current_comm(&data.comm, sizeof(data.comm));
+        data.pid = pid;
+        data.tid = tid;
+        data.old_nid = old_nid;
+        data.new_nid = new_nid;
+
+        events.perf_submit(args, &data, sizeof(data));
+    }
+
+update:
+    numaid_info.update(&tid, &val);
+    return 0;
+}
+"""
+
+if args.pid:
+    bpf_text = bpf_text.replace('FILTER_PID', 'pid != %s' % args.pid)
+else:
+    # always skip PID=0
+    bpf_text = bpf_text.replace('FILTER_PID', 'pid == 0')
+
+if args.tid:
+    bpf_text = bpf_text.replace('FILTER_TID', 'tid != %s' % args.tid)
+else:
+    # always skip TID=0
+    bpf_text = bpf_text.replace('FILTER_TID', 'tid == 0')
+
+# process event
+def print_event(cpu, data, size):
+    event = b["events"].event(data)
+
+    # Filter events by comm
+    if args.comm:
+        if not args.comm == event.comm.decode('utf-8', 'replace'):
+            return
+
+    print("%-8s %-8d %-8d %-8d -> %-8d %-8s" %
+        (strftime("%H:%M:%S"),
+        event.pid,
+        event.tid,
+        event.old_nid,
+        event.new_nid,
+        event.comm))
+
+
+b = BPF(text=bpf_text)
+
+print("Tracing task NUMA switch...")
+print("%-8s %-8s %-8s %-8s    %-8s %-8s" %
+    ("TIME", "PID", "TID", "SRC_NID", "DST_NID", "COMM"))
+
+b["events"].open_perf_buffer(print_event)
+while 1:
+    try:
+        b.perf_buffer_poll()
+    except KeyboardInterrupt:
+        exit()
+

tools/numasched_example.txt

@@ -0,0 +1,55 @@
+Demonstrations of numasched.py, the Linux eBPF/bcc version.
+
+This example trace the task switch numa. Some example output:
+
+NUMA Information:
+
+  $ numactl --hardware
+  available: 4 nodes (0-3)
+  node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
+  node 0 size: 97373 MB
+  node 0 free: 1756 MB
+  node 1 cpus: 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
+  node 1 size: 98192 MB
+  node 1 free: 1269 MB
+  node 2 cpus: 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
+  node 2 size: 98192 MB
+  node 2 free: 4811 MB
+  node 3 cpus: 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
+  node 3 size: 98135 MB
+  node 3 free: 1617 MB
+  node distances:
+  node   0   1   2   3
+    0:  10  12  20  22
+    1:  12  10  22  24
+    2:  20  22  10  12
+    3:  22  24  12  10
+
+
+Terminal 1, start a task running on NUMA0:
+
+  $ taskset -c 1 yes >/dev/null
+
+
+Terminal 2, start tracing:
+
+  $ sudo ./numasched.py
+
+
+Terminal 3
+
+  # taskset 'yes' task to NUMA1(cpu=24):
+  $ taskset -p 0x1000000 $(pidof yes)
+
+  # taskset 'yes' task to NUMA2(cpu=48):
+  $ taskset -p 0x1000000000000 $(pidof yes)
+
+
+Then, Terminal 2 shows:
+
+  $ sudo ./numasched.py
+  Tracing task NUMA switch...
+  TIME     PID      TID      SRC_NID     DST_NID  COMM
+  20:55:35 355842   355842   0        -> 1        b'yes'
+  20:55:50 355842   355842   1        -> 2        b'yes'
+