three programs.

and tears. today was hard.

Signed-off-by: vsoch <[email protected]>

Author: vsoch

README.md

@@ -540,22 +540,80 @@ For this setup, you'll see `flux submit` so the jobs will run at the same time o
 
 ##### 3. Monitor with BCC
 
-This setup will deploy a sidecar and monitor different interacts with bcc.
+This setup will deploy a sidecar and monitor different interacts with bcc. We have several programs that help to understand tcp, file open/closes, or futex wait times. Although you can filter to a cgroup or command, for the default we allow all containers in the pod to be seen. It generates a lot more data, but is interesting. This might need to be tweaked for actual experiments if the data is too big. The default is file open/close (you could leave out `monitor_program`) below:
 
 ```bash
 helm install \
+  --set experiment.monitor_program=open-close \
   --set experiment.monitor=true \
   --set minicluster.save_logs=true \
   lammps ./lammps-reax
 ```
 
-You'll need to look at the logs to see the sidecar vs. lammps.
+<details>
+
+```console
+Looking for /opt/programs/open-close/ebpf-collect.c
+Starting eBPF (Tracepoint for open entry).
+
+Start Indicator file defined '/mnt/flux/start_ebpf_collection'. Waiting.
+{"event": "OPEN", "command": "python3", "retval": 12, "ts_sec": 779.540036095, "tgid": 0, "tid": 14554, "ppid": 14554, "cgroup_id": 0, "filename": "/sys/bus/event_source/devices/kprobe/type"}
+{"event": "OPEN", "command": "python3", "retval": 12, "ts_sec": 779.540051234, "tgid": 0, "tid": 14554, "ppid": 14554, "cgroup_id": 0, "filename": "/sys/bus/event_source/devices/kprobe/format/retprobe"}
+{"event": "OPEN", "command": "containerd", "retval": 193, "ts_sec": 779.629315113, "tgid": 0, "tid": 3600, "ppid": 3618, "cgroup_id": 0, "filename": "/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/204/fs"}
+{"event": "CLOSE", "command": "containerd", "retval": 0, "ts_sec": 779.629342785, "tgid": 1, "tid": 3600, "ppid": 3618, "cgroup_id": 6520}
+...
+{"event": "OPEN", "command": "touch", "retval": 3, "ts_sec": 803.043308743, "tgid": 0, "tid": 14883, "ppid": 14883, "cgroup_id": 3257288213055174703, "filename": "/usr/lib/locale/C.utf8/LC_NUMERIC"}
+{"event": "CLOSE", "command": "touch", "retval": 0, "ts_sec": 803.043310733, "tgid": 14414, "tid": 14883, "ppid": 14883, "cgroup_id": 13176}
+{"event": "OPEN", "command": "touch", "retval": 3, "ts_sec": 803.043316595, "tgid": 0, "tid": 14883, "ppid": 14883, "cgroup_id": 3257288213055174703, "filename": "/usr/lib/locale/C.utf8/LC_CTYPE"}
+{"event": "CLOSE", "command": "touch", "retval": 0, "ts_sec": 803.043318514, "tgid": 14414, "tid": 14883, "ppid": 14883, "cgroup_id": 13176}
+{"event": "OPEN", "command": "touch", "retval": 3, "ts_sec": 803.043359627, "tgid": 0, "tid": 14883, "ppid": 14883, "cgroup_id": 3257288213055174703, "filename": "/mnt/flux/stop_ebpf_collection"}
+{"event": "CLOSE", "command": "touch", "retval": 0, "ts_sec": 803.043360931, "tgid": 14414, "tid": 14883, "ppid": 14883, "cgroup_id": 13176}
+
+Indicator file '/mnt/flux/stop_ebpf_collection' found. Stopping.
+Cleaning up BPF resources...
+```
+
+</details>
+
+Here is how to see futex wait times.
+
+```bash
+helm install \
+  --set experiment.monitor_program=futex \
+  --set experiment.monitor=true \
+  --set minicluster.save_logs=true \
+  lammps ./lammps-reax
+```
+
+This likely needs to be consolidated (it's a lot of data). Here is an example.
+
+```bash
+{"event_type": "FUTEX_WAIT_END", "timestamp_sec": 3891.635076022, "tgid": 34931, "tid": 35146, "comm": "containerd-shim", "cgroup_id": 6520, "futex_op_full": 128, "futex_op_str": "FUTEX_WAIT_PRIVATE", "wait_duration_ns": 21462, "wait_duration_human": "21.46us"}
+{"event_type": "FUTEX_WAIT_END", "timestamp_sec": 3891.635086712, "tgid": 34931, "tid": 35388, "comm": "containerd-shim", "cgroup_id": 6520, "futex_op_full": 128, "futex_op_str": "FUTEX_WAIT_PRIVATE", "wait_duration_ns": 42892, "wait_duration_human": "42.89us"}
+{"event_type": "FUTEX_WAIT_END", "timestamp_sec": 3891.635102633, "tgid": 3600, "tid": 3602, "comm": "containerd", "cgroup_id": 6520, "futex_op_full": 128, "futex_op_str": "FUTEX_WAIT_PRIVATE", "wait_duration_ns": 16693, "wait_duration_human": "16.69us"}
+{"event_type": "FUTEX_WAIT_END", "timestamp_sec": 3891.635107428, "tgid": 3600, "tid": 6823, "comm": "containerd", "cgroup_id": 6520, "futex_op_full": 128, "futex_op_str": "FUTEX_WAIT_PRIVATE", "wait_duration_ns": 34781, "wait_duration_human": "34.78us"}
+```
+
+Finally, here is tcp
+
+```bash
+helm install \
+  --set experiment.monitor_program=tcp \
+  --set experiment.monitor=true \
+  --set minicluster.save_logs=true \
+  lammps ./lammps-reax
+```
+
+Here is example data:
 
 ```bash
-kubeclt lo
+{"event_type": "RECV", "timestamp_sec": 3978.855691216, "tgid": 36693, "tid": 36693, "comm": "lmp", "cgroup_id": 19947, "fd": 11, "bytes": 20, "bytes_human": "20", "duration_ns": 13430, "duration_human": "13.43us"}
+{"event_type": "RECV", "timestamp_sec": 3978.855698151, "tgid": 36693, "tid": 36693, "comm": "lmp", "cgroup_id": 19947, "fd": 11, "bytes": 20, "bytes_human": "20", "duration_ns": 1920, "duration_human": "1.92us"}
+{"event_type": "RECV", "timestamp_sec": 3978.855731232, "tgid": 36693, "tid": 36693, "comm": "lmp", "cgroup_id": 19947, "fd": 11, "bytes": 20, "bytes_human": "20", "duration_ns": 12363, "duration_human": "12.36us"}
+{"event_type": "RECV", "timestamp_sec": 3978.855737251, "tgid": 36693, "tid": 36693, "comm": "lmp", "cgroup_id":
 ```
 
-Try changing the command:
+Also try changing the command entirely.
 
 ```bash
 helm install \

base-template/docker/bcc-sidecar/programs/futex/ebpf-collect.c

@@ -0,0 +1,157 @@
+#include <uapi/linux/ptrace.h>
+#include <linux/sched.h>
+// Do NOT include <linux/futex.h> here as it can cause compilation issues with BCC
+
+// Define FUTEX constants directly in BPF C code
+// These values are from standard Linux UAPI headers (e.g., <uapi/linux/futex.h>)
+#ifndef FUTEX_WAIT
+#define FUTEX_WAIT		0
+#endif
+#ifndef FUTEX_WAKE
+#define FUTEX_WAKE		1
+#endif
+// Add other FUTEX_ op codes if we want to trace them, e.g.
+// #ifndef FUTEX_LOCK_PI
+// #define FUTEX_LOCK_PI		6
+// #endif
+
+// Flags for futex op
+#ifndef FUTEX_PRIVATE_FLAG
+#define FUTEX_PRIVATE_FLAG	128
+#endif
+#ifndef FUTEX_CLOCK_REALTIME
+#define FUTEX_CLOCK_REALTIME	256 // Note: If this flag is set, the 'op' argument structure changes for timeout
+#endif
+
+// Mask to get the command part of the futex op
+// FUTEX_CMD_MASK should correctly isolate the command part from flags.
+// A common way to get the command is (op & 0x7F) or (op & ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME))
+// The definition of FUTEX_CMD_MASK here assumes we only care about private/realtime flags for masking.
+// If other flags are present in the lower bits that are not part of the command, this might need adjustment.
+// For simplicity, we often just check `(op & 0x7F) == FUTEX_WAIT` or `op == FUTEX_WAIT` or `op == FUTEX_WAIT_PRIVATE`.
+// Let's use a simpler direct check for FUTEX_WAIT and FUTEX_WAIT_PRIVATE.
+#ifndef FUTEX_WAIT_PRIVATE
+#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)
+#endif
+
+
+#define TASK_COMM_LEN_EBPF 16
+
+enum event_type {
+    EVENT_FUTEX_WAIT_COMPLETED = 0,
+};
+
+struct data_t {
+    u64 timestamp_ns;
+    u32 tgid;
+    u32 tid;
+    // u32 ppid; // Add back if needed and fetched in BPF
+    u64 cgroup_id;
+    char comm[TASK_COMM_LEN_EBPF];
+    enum event_type type;
+    int futex_op_full;      // The original futex op argument (including flags)
+    u64 wait_duration_ns;
+};
+BPF_RINGBUF_OUTPUT(events, 8);
+
+BPF_HASH(futex_start_times, u32, u64); // Key: TID, Value: Start timestamp
+BPF_HASH(futex_ops_temp, u32, int);    // Key: TID, Value: Original futex_op_full from enter
+
+// --- Debugging (Optional) ---
+enum debug_stage {
+    DBG_FUTEX_ENTER_TRACKING = 400,
+    DBG_FUTEX_ENTER_NOT_TRACKING = 401,
+    DBG_FUTEX_EXIT_FOUND_START = 402,
+    DBG_FUTEX_EXIT_NO_START = 403,
+    DBG_FUTEX_SUBMITTED = 404,
+    DBG_FUTEX_RESERVE_FAIL = 405
+};
+struct debug_event_t {
+    u32 id_tid; // Using u32 for TID
+    enum debug_stage stage;
+    long val1_op;
+    long val2_duration_or_misc;
+};
+BPF_RINGBUF_OUTPUT(debug_events_rb, 4);
+
+static __always_inline void send_futex_debug_event(u32 current_tid, enum debug_stage stage, long v1, long v2) {
+    struct debug_event_t *dbg_evt = debug_events_rb.ringbuf_reserve(sizeof(struct debug_event_t));
+    if (dbg_evt) {
+        dbg_evt->id_tid = current_tid;
+        dbg_evt->stage = stage;
+        dbg_evt->val1_op = v1;
+        dbg_evt->val2_duration_or_misc = v2;
+        debug_events_rb.ringbuf_submit(dbg_evt, 0);
+    }
+}
+// --- End Debugging ---
+
+TRACEPOINT_PROBE(syscalls, sys_enter_futex) {
+    u64 pid_tgid = bpf_get_current_pid_tgid();
+    u32 tid = (u32)pid_tgid;
+
+    // 'args->op' is how BCC TRACEPOINT_PROBE typically exposes tracepoint arguments
+    // that match the 'op' field in /sys/kernel/debug/tracing/events/syscalls/sys_enter_futex/format
+    int futex_op_full = args->op;
+    int futex_op_cmd = futex_op_full & ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME); // Get command part
+
+    // We are interested in FUTEX_WAIT or FUTEX_WAIT_PRIVATE
+    if (futex_op_cmd == FUTEX_WAIT) {
+        u64 start_time_ns = bpf_ktime_get_ns();
+        futex_start_times.update(&tid, &start_time_ns);
+        futex_ops_temp.update(&tid, &futex_op_full); // Store the original op
+        // send_futex_debug_event(tid, DBG_FUTEX_ENTER_TRACKING, futex_op_full, 0);
+    } else {
+        // send_futex_debug_event(tid, DBG_FUTEX_ENTER_NOT_TRACKING, futex_op_full, 0);
+    }
+    return 0;
+}
+
+TRACEPOINT_PROBE(syscalls, sys_exit_futex) {
+    u64 pid_tgid = bpf_get_current_pid_tgid();
+    u32 tgid = pid_tgid >> 32;
+    u32 tid = (u32)pid_tgid;
+
+    u64 *start_time_ns_ptr;
+    start_time_ns_ptr = futex_start_times.lookup(&tid);
+
+    if (start_time_ns_ptr) {
+        u64 end_time_ns = bpf_ktime_get_ns();
+        u64 duration_ns = end_time_ns - *start_time_ns_ptr;
+        
+        int *original_op_ptr = futex_ops_temp.lookup(&tid);
+        int original_op = -1; // Default if not found (shouldn't happen if start_time was found)
+        if (original_op_ptr) {
+            original_op = *original_op_ptr;
+        }
+
+        futex_start_times.delete(&tid);
+        futex_ops_temp.delete(&tid);
+
+        // send_futex_debug_event(tid, DBG_FUTEX_EXIT_FOUND_START, original_op, duration_ns);
+
+        struct data_t *event_data_ptr = events.ringbuf_reserve(sizeof(*event_data_ptr));
+        if (!event_data_ptr) {
+            // send_futex_debug_event(tid, DBG_FUTEX_RESERVE_FAIL, original_op, 0);
+            return 0;
+        }
+
+        event_data_ptr->timestamp_ns = end_time_ns;
+        event_data_ptr->tgid = tgid;
+        event_data_ptr->tid = tid;
+        event_data_ptr->cgroup_id = bpf_get_current_cgroup_id();
+        bpf_get_current_comm(&event_data_ptr->comm, sizeof(event_data_ptr->comm));
+        event_data_ptr->comm[TASK_COMM_LEN_EBPF - 1] = '\0';
+        event_data_ptr->type = EVENT_FUTEX_WAIT_COMPLETED;
+        event_data_ptr->futex_op_full = original_op;
+        event_data_ptr->wait_duration_ns = duration_ns;
+
+        events.ringbuf_submit(event_data_ptr, 0);
+        // send_futex_debug_event(tid, DBG_FUTEX_SUBMITTED, original_op, duration_ns);
+
+    } else {
+         // send_futex_debug_event(tid, DBG_FUTEX_EXIT_NO_START, args->ret, 0); // args->ret is syscall return
+    }
+    return 0;
+}
+