Get custom labels working with clang-17 and 4.19 kernel

Author: gnurizen

.github/workflows/unit-test-on-pull-request.yml

@@ -120,6 +120,7 @@ jobs:
           # https://github.com/cilium/ci-kernels/pkgs/container/ci-kernels/versions?filters%5Bversion_type%5D=tagged
 
           # AMD64
+          - { target_arch: amd64, kernel: 4.19.314 }
           - { target_arch: amd64, kernel: 5.4.276 }
           - { target_arch: amd64, kernel: 5.10.217 }
           - { target_arch: amd64, kernel: 5.15.159 }

support/ebpf/Makefile

@@ -1,7 +1,7 @@
 SHELL ?= bash
-BPF_CLANG ?= clang-16
-BPF_LINK ?= llvm-link-16
-LLC ?= llc-16
+BPF_CLANG ?= clang-17
+BPF_LINK ?= llvm-link-17
+LLC ?= llc-17
 
 DEBUG_FLAGS = -DOPTI_DEBUG -g
 

support/ebpf/bpfdefs.h

@@ -157,14 +157,4 @@ static long (*bpf_perf_prog_read_value)(struct pt_regs *ctx, struct bpf_perf_eve
 
 #endif // !TESTING_COREDUMP
 
-// HACK: On failure, bpf_perf_prog_read_value() zeroes the buffer. We ensure that this always
-// fail with a compile time assert that ensures that the struct size is different to the size
-// of the expected structure.
-#define bpf_large_memzero(_d, _l)                                                                                        \
-    ({                                                                                                                   \
-        _Static_assert(_l != sizeof(struct bpf_perf_event_value), "stack size must be different to the valid argument"); \
-        bpf_perf_prog_read_value(ctx, _d, _l);                                                                           \
-    })
-
-
 #endif // OPTI_BPFDEFS_H

support/ebpf/extmaps.h

@@ -17,6 +17,7 @@ extern bpf_map_def inhibit_events;
 extern bpf_map_def interpreter_offsets;
 extern bpf_map_def system_config;
 extern bpf_map_def trace_events;
+extern bpf_map_def go_procs;
 
 #if defined(TESTING_COREDUMP)
 

support/ebpf/go_labels.ebpf.c

@@ -0,0 +1,160 @@
+// This file contains the code and map definitions that are shared between
+// the tracers, as well as a dispatcher program that can be attached to a
+// perf event and will call the appropriate tracer for a given process
+
+#include "bpfdefs.h"
+#include "util.h"
+#include "hash.h"
+#include "kernel.h"
+#include "tracemgmt.h"
+#include "tsd.h"
+#include "types.h"
+
+static inline __attribute__((__always_inline__))
+void process_value(GoMapBucket *map_value, CustomLabelsArray *out, unsigned i) {
+    if (map_value->tophash[i] == 0)
+        return;
+    if (out->len >= MAX_CUSTOM_LABELS)
+        return;
+    CustomLabel *lbl = &out->labels[out->len];
+    if (map_value->keys[i].str != NULL) {
+        long res = bpf_probe_read_user(lbl->key.key_bytes, CUSTOM_LABEL_MAX_KEY_LEN, map_value->keys[i].str);
+        if (res) {
+            DEBUG_PRINT("cl: failed to read key for custom label (%lx): %ld", (unsigned long) map_value->keys[i].str, res);
+            return;
+        }
+        res = bpf_probe_read_user(lbl->val.val_bytes, CUSTOM_LABEL_MAX_VAL_LEN, map_value->values[i].str);
+        if (res) {
+            DEBUG_PRINT("cl: failed to read value for custom label: %ld", res);
+            return;
+        }
+        lbl->key_len = map_value->keys[i].len;
+        lbl->val_len = map_value->values[i].len;
+    }
+    out->len++;
+}
+
+static inline __attribute__((__always_inline__))
+bool process_bucket(PerCPURecord *record, void *label_buckets, int j) {
+    CustomLabelsArray *out = &record->customLabelsState.cla;
+    GoMapBucket *map_value = &record->goMapBucket;
+    long res = bpf_probe_read(map_value, sizeof(GoMapBucket), label_buckets + (j * sizeof(GoMapBucket)));
+    if (res < 0) {
+        return false;
+    }
+
+    process_value(map_value, out, 0);
+    process_value(map_value, out, 1);
+    process_value(map_value, out, 2);
+    process_value(map_value, out, 3);
+    process_value(map_value, out, 4);
+    process_value(map_value, out, 5);
+    process_value(map_value, out, 6);
+    process_value(map_value, out, 7);
+
+    return false;
+}
+
+// Go processes store the current goroutine in thread local store. From there
+// this reads the g (aka goroutine) struct, then the m (the actual operating
+// system thread) of that goroutine, and finally curg (current goroutine). This
+// chain is necessary because getg().m.curg points to the current user g
+// assigned to the thread (curg == getg() when not on the system stack). curg
+// may be nil if there is no user g, such as when running in the scheduler. If
+// curg is nil, then g is either a system stack (called g0) or a signal handler
+// g (gsignal). Neither one will ever have label.
+static inline __attribute__((__always_inline__))
+bool get_go_custom_labels(struct pt_regs *ctx, PerCPURecord *record, GoCustomLabelsOffsets *offs) {
+    long res;
+
+    size_t curg_ptr_addr;
+    res = bpf_probe_read_user(&curg_ptr_addr, sizeof(void *), (void *)(record->customLabelsState.go_m_ptr + offs->curg));
+    if (res < 0) {
+        DEBUG_PRINT("cl: failed to read value for m_ptr->curg: %ld", res);
+        return false;
+    }
+
+    void *labels_map_ptr_ptr;
+    res = bpf_probe_read_user(&labels_map_ptr_ptr, sizeof(void *), (void *)(curg_ptr_addr + offs->labels));
+    if (res < 0) {
+        DEBUG_PRINT("cl: failed to read value for curg->labels (%lx->%lx): %ld", (unsigned long)curg_ptr_addr,
+          (unsigned long) offs->labels, res);
+        return false;
+    }
+
+    void *labels_map_ptr;
+    res = bpf_probe_read(&labels_map_ptr, sizeof(labels_map_ptr), labels_map_ptr_ptr);
+    if (res < 0) {
+        DEBUG_PRINT("cl: failed to read value for labels_map_ptr (%lx): %ld", (unsigned long)labels_map_ptr_ptr, res);
+        return false;
+    }
+
+    u64 labels_count = 0;
+    res = bpf_probe_read(&labels_count, sizeof(labels_count), labels_map_ptr + offs->hmap_count);
+    if (res < 0) {
+        DEBUG_PRINT("cl: failed to read value for labels_count: %ld", res);
+        return false;
+    }
+    if (labels_count == 0) {
+        DEBUG_PRINT("cl: no labels");
+        return false;
+    }
+
+    unsigned char log_2_bucket_count;
+    res = bpf_probe_read(&log_2_bucket_count, sizeof(log_2_bucket_count), labels_map_ptr + offs->hmap_log2_bucket_count);
+    if (res < 0) {
+        DEBUG_PRINT("cl: failed to read value for bucket_count: %ld", res);
+        return false;
+    }
+    void *label_buckets;
+    res = bpf_probe_read(&label_buckets, sizeof(label_buckets), labels_map_ptr + offs->hmap_buckets);
+    if (res < 0) {
+        DEBUG_PRINT("cl: failed to read value for label_buckets: %ld", res);
+        return false;
+    }
+
+    // Manually unroll loop to support 4.19 kernel, auto unroll doesn't work as well
+    // and we can't support as many buckets.
+    u64 bucket_count = MIN(MAX_CUSTOM_LABELS, 1 << log_2_bucket_count);
+    switch (bucket_count) {
+        case 14: if (process_bucket(record, label_buckets, 13)) return true;
+        case 13: if (process_bucket(record, label_buckets, 12)) return true;
+        case 12: if (process_bucket(record, label_buckets, 11)) return true;
+        case 11: if (process_bucket(record, label_buckets, 10)) return true;
+        case 10: if (process_bucket(record, label_buckets, 9)) return true;
+        case 9: if (process_bucket(record, label_buckets, 8)) return true;
+        case 8: if (process_bucket(record, label_buckets, 7)) return true;
+        case 7: if (process_bucket(record, label_buckets, 6)) return true;
+        case 6: if (process_bucket(record, label_buckets, 5)) return true;
+        case 5: if (process_bucket(record, label_buckets, 4)) return true;
+        case 4: if (process_bucket(record, label_buckets, 3)) return true;
+        case 3: if (process_bucket(record, label_buckets, 2)) return true;
+        case 2: if (process_bucket(record, label_buckets, 1)) return true;
+        case 1: if (process_bucket(record, label_buckets, 0)) return true;
+    }
+
+    return false;
+}
+
+
+SEC("perf_event/go_labels")
+int go_labels(struct pt_regs *ctx) {
+  PerCPURecord *record = get_per_cpu_record();
+  if (!record)
+    return -1;
+
+  u32 pid = record->trace.pid;
+  GoCustomLabelsOffsets *offsets = bpf_map_lookup_elem(&go_procs, &pid);
+  if (!offsets) {
+    DEBUG_PRINT("cl: no offsets, %d not recognized as a go binary", pid);
+    return -1;
+  }
+  DEBUG_PRINT("cl: go offsets found, %d recognized as a go binary: m_ptr: %lx", pid, (unsigned long)record->customLabelsState.go_m_ptr);
+  bool success = get_go_custom_labels(ctx, record, offsets);
+  if (!success) {
+    increment_metric(metricID_UnwindGoCustomLabelsFailures);
+  }
+
+  tail_call(ctx, PROG_UNWIND_STOP);
+  return 0;
+}

support/ebpf/hash.h

@@ -3,90 +3,33 @@
 
 #include "types.h"
 
-#define ROUNDUP_8(x) ((x + 7) & ~7)
-static inline __attribute__((__always_inline__))
-bool hash_custom_labels(CustomLabelsArray *lbls, int seed, u64 *out) {
-    // apply murmurhash2 as though this is an array of
-    // the number of labels (8 bytes), followed by all the key/val lengths,
-    // followed by all the keys/vals.
-    const u64 m = 0xc6a4a7935bd1e995LLU;
-    const int r = 47;
-
-    int len = 8;
-    for (int i = 0; i < MAX_CUSTOM_LABELS; ++i) {
-        if (i >= lbls->len)
-            break;
-        len += 8;
-        len += ROUNDUP_8(lbls->labels[i].key_len);
-        len += ROUNDUP_8(lbls->labels[i].val_len);
-    }
-
-    u64 h = seed ^ (len * m);
-
-    // hash the number of labels
-    {
-        u64 k = lbls->len;
-        k *= m;
-        k ^= k >> r;
-        k *= m;
-
-        h ^= k;
-        h *= m;
-    }
-
-    // hash each k/v len
-    for (int i = 0; i < MAX_CUSTOM_LABELS; ++i) {
-        // force clang not to unroll the loop by hiding the value of i.
-        // Unrolling this loop confuses the verifier.
-        asm volatile("" : "=r"(i) : "0"(i));
-        if (i >= lbls->len)
-            break;
-        u64 k = (((u64)lbls->labels[i].key_len) << 32) | ((u64)lbls->labels[i].val_len);
-        k *= m;
-        k ^= k >> r;
-        k *= m;
+#define M 0xc6a4a7935bd1e995LLU
 
-        h ^= k;
-        h *= m;
+static inline __attribute__((__always_inline__))
+u64 clear_or_hash_custom_labels(CustomLabelsArray *lbls, bool clear) {
+    u64 h = lbls->len * M;
+    u64 *bits = (u64 *)lbls;
+#pragma unroll
+    for (int i=0; i < sizeof(CustomLabelsArray)/8; i++) {
+      if (clear) {
+        bits[i] = 0;
+      } else {
+        h ^= bits[i];
+        h *= M;
+      }
     }
 
-    // hash each k/v
-    for (int i = 0; i < MAX_CUSTOM_LABELS; ++i) {
-        if (i >= lbls->len)
-            break;
-        CustomLabel *lbl = &lbls->labels[i];
-        u64 kl = ROUNDUP_8(lbl->key_len);
-        for (int j = 0; j < CUSTOM_LABEL_MAX_VAL_LEN / 8; ++j) {
-            if (j >= kl)
-                return false;
-            u64 k = lbl->key.key_u64[j];
-            k *= m;
-            k ^= k >> r;
-            k *= m;
-
-            h ^= k;
-            h *= m;
-        }
-        u64 vl = ROUNDUP_8(lbl->val_len);
-        for (int j = 0; j < CUSTOM_LABEL_MAX_VAL_LEN / 8; ++j) {
-            if (j >= vl)
-                return false;
-            u64 k = lbl->val.val_u64[j];
-            k *= m;
-            k ^= k >> r;
-            k *= m;
-
-            h ^= k;
-            h *= m;
-        }
-    }
+    return h;
+}
 
-    h ^= h >> r;
-    h *= m;
-    h ^= h >> r;
+static inline __attribute__((__always_inline__))
+void clear_custom_labels(CustomLabelsArray *lbls) {
+  clear_or_hash_custom_labels(lbls, true);
+}
 
-    *out = h;
-    return true;
+static inline __attribute__((__always_inline__))
+u64 hash_custom_labels(CustomLabelsArray *lbls) {
+  return clear_or_hash_custom_labels(lbls, false);
 }
 
-#endif
+#endif  // OPTI_HASH_H