Skip to content

Commit 09bf384

Browse files
Bertkrisztianfekete
Bert
and
krisztianfekete
authored
Adds histogram prometheus metrics (#112)
* histogram * user can pass buckets * bucket size arg parsing works * code cleanup and added exmaple * comments * made example more comprehensive and add support for chars * Update pkg/stats/stats.go Co-authored-by: Krisztian Fekete <103492698+krisztianfekete@users.noreply.github.com> * Update pkg/stats/stats.go Co-authored-by: Krisztian Fekete <103492698+krisztianfekete@users.noreply.github.com> * Update pkg/stats/stats.go Co-authored-by: Krisztian Fekete <103492698+krisztianfekete@users.noreply.github.com> * fix compiler error * added GPL license identifier to example * added attribution comment to example * Update pkg/cli/internal/commands/run/run.go Co-authored-by: Krisztian Fekete <103492698+krisztianfekete@users.noreply.github.com> * fixed bug where couldn't set histogram key --------- Co-authored-by: Krisztian Fekete <103492698+krisztianfekete@users.noreply.github.com>
1 parent c2422b5 commit 09bf384

File tree

5 files changed

+372
-18
lines changed

5 files changed

+372
-18
lines changed

‎examples/nfsstats/nfsstats.c‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
// Based on: https://github.com/iovisor/bcc/blob/master/libbpf-tools/fsslower.bpf.c
2+
// SPDX-License-Identifier: GPL-2.0
3+
4+
#include "vmlinux.h"
5+
#include "bpf/bpf_helpers.h"
6+
#include "bpf/bpf_core_read.h"
7+
#include "bpf/bpf_tracing.h"
8+
#include "solo_types.h"
9+
10+
// Example for tracing NFS operation file duration using histogram metrics
11+
12+
char __license[] SEC("license") = "Dual MIT/GPL";
13+
14+
struct event {
15+
char fname[255];
16+
char op;
17+
u64 le;
18+
};
19+
20+
struct event_start {
21+
u64 ts;
22+
struct file *fp;
23+
};
24+
25+
struct {
26+
__uint(type, BPF_MAP_TYPE_HASH);
27+
__uint(max_entries, 4096);
28+
__type(key, u32);
29+
__type(value, struct event_start);
30+
} start SEC(".maps");
31+
32+
struct {
33+
__uint(type, BPF_MAP_TYPE_RINGBUF);
34+
__uint(max_entries, 1 << 24);
35+
__type(value, struct event);
36+
} hist_nfs_op_time_us SEC(".maps");
37+
38+
static __always_inline int
39+
probe_entry(struct file *fp)
40+
{
41+
bpf_printk("nfs_file_read happened");
42+
43+
struct event_start evt = {};
44+
45+
u32 tgid = bpf_get_current_pid_tgid() >> 32;
46+
u64 ts = bpf_ktime_get_ns();
47+
48+
evt.ts = ts;
49+
evt.fp = fp;
50+
bpf_map_update_elem(&start, &tgid, &evt, 0);
51+
52+
return 0;
53+
}
54+
55+
static __always_inline int
56+
probe_exit(char op) {
57+
struct event evt = {};
58+
struct file *fp;
59+
struct dentry *dentry;
60+
const __u8 *file_name;
61+
62+
u32 tgid = bpf_get_current_pid_tgid() >> 32;
63+
struct event_start *rs;
64+
65+
rs = bpf_map_lookup_elem(&start, &tgid);
66+
if (!rs)
67+
return 0;
68+
69+
u64 ts = bpf_ktime_get_ns();
70+
u64 duration = (ts - rs->ts) / 1000;
71+
72+
bpf_printk("nfs operation duration: %lld", duration);
73+
74+
evt.le = duration;
75+
evt.op = op;
76+
77+
// decode filename
78+
fp = rs->fp;
79+
dentry = BPF_CORE_READ(fp, f_path.dentry);
80+
file_name = BPF_CORE_READ(dentry, d_name.name);
81+
bpf_probe_read_kernel_str(evt.fname, sizeof(evt.fname), file_name);
82+
bpf_printk("nfs op file_name: %s", evt.fname);
83+
84+
struct event *ring_val;
85+
ring_val = bpf_ringbuf_reserve(&hist_nfs_op_time_us, sizeof(evt), 0);
86+
if (!ring_val)
87+
return 0;
88+
89+
memcpy(ring_val, &evt, sizeof(evt));
90+
bpf_ringbuf_submit(ring_val, 0);
91+
}
92+
93+
SEC("kprobe/nfs_file_read")
94+
int BPF_KPROBE(nfs_file_read, struct kiocb *iocb) {
95+
bpf_printk("nfs_file_read happened");
96+
struct file *fp = BPF_CORE_READ(iocb, ki_filp);
97+
return probe_entry(fp);
98+
}
99+
100+
SEC("kretprobe/nfs_file_read")
101+
int BPF_KRETPROBE(nfs_file_read_ret, ssize_t ret) {
102+
bpf_printk("nfs_file_read returtned");
103+
return probe_exit('r');
104+
}
105+
106+
SEC("kprobe/nfs_file_write")
107+
int BPF_KPROBE(nfs_file_write, struct kiocb *iocb) {
108+
bpf_printk("nfs_file_write happened");
109+
struct file *fp = BPF_CORE_READ(iocb, ki_filp);
110+
return probe_entry(fp);
111+
}
112+
113+
SEC("kretprobe/nfs_file_write")
114+
int BPF_KRETPROBE(nfs_file_write_ret, ssize_t ret) {
115+
bpf_printk("nfs_file_write returtned");
116+
return probe_exit('w');
117+
}
118+
119+
SEC("kprobe/nfs_file_open")
120+
int BPF_KPROBE(nfs_file_open, struct inode *inode, struct file *fp) {
121+
bpf_printk("nfs_file_open happened");
122+
return probe_entry(fp);
123+
}
124+
125+
SEC("kretprobe/nfs_file_open")
126+
int BPF_KRETPROBE(nfs_file_open_ret, struct file *fp) {
127+
bpf_printk("nfs_file_open returtned");
128+
return probe_exit('o');
129+
}

‎pkg/cli/internal/commands/run/run.go‎

Lines changed: 81 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import (
77
"io"
88
"os"
99
"os/signal"
10+
"strconv"
11+
"strings"
1012
"syscall"
1113

1214
"github.com/cilium/ebpf/rlimit"
@@ -27,14 +29,19 @@ import (
2729
type runOptions struct {
2830
general *options.GeneralOptions
2931

30-
debug bool
31-
filter []string
32-
notty bool
33-
pinMaps string
34-
pinProgs string
35-
promPort uint32
32+
debug bool
33+
filter []string
34+
histBuckets []string
35+
histValueKey []string
36+
notty bool
37+
pinMaps string
38+
pinProgs string
39+
promPort uint32
3640
}
3741

42+
const histBucketsDescription string = "Buckets to use for histogram maps. Format is \"map_name,<buckets_limits>\"" +
43+
"where <buckets_limits> is a comma separated list of bucket limits. For example: \"events,[1,2,3,4,5]\""
44+
3845
const filterDescription string = "Filter to apply to output from maps. Format is \"map_name,key_name,regex\" " +
3946
"You can define a filter per map, if more than one defined, the last defined filter will take precedence"
4047

@@ -43,6 +50,8 @@ var stopper chan os.Signal
4350
func addToFlags(flags *pflag.FlagSet, opts *runOptions) {
4451
flags.BoolVarP(&opts.debug, "debug", "d", false, "Create a log file 'debug.log' that provides debug logs of loader and TUI execution")
4552
flags.StringSliceVarP(&opts.filter, "filter", "f", []string{}, filterDescription)
53+
flags.StringArrayVarP(&opts.histBuckets, "hist-buckets", "b", []string{}, histBucketsDescription)
54+
flags.StringArrayVarP(&opts.histValueKey, "hist-value-key", "k", []string{}, "Key to use for histogram maps. Format is \"map_name,key_name\"")
4655
flags.BoolVar(&opts.notty, "no-tty", false, "Set to true for running without a tty allocated, so no interaction will be expected or rich output will done")
4756
flags.StringVar(&opts.pinMaps, "pin-maps", "", "Directory to pin maps to, left unpinned if empty")
4857
flags.StringVar(&opts.pinProgs, "pin-progs", "", "Directory to pin progs to, left unpinned if empty")
@@ -72,6 +81,10 @@ $ bee run -f="events,comm,node" ghcr.io/solo-io/bumblebee/opensnoop:0.0.7
7281
7382
To run with multiple filters, use the --filter (or -f) flag multiple times:
7483
$ bee run -f="events_hash,daddr,1.1.1.1" -f="events_ring,daddr,1.1.1.1" ghcr.io/solo-io/bumblebee/tcpconnect:0.0.7
84+
85+
If your program has histogram output, you can supply the buckets using --buckets (or -b) flag:
86+
TODO(albertlockett) add a program w/ histogram buckets as example
87+
$ bee run -b="events,[1,2,3,4,5]" ghcr.io/solo-io/bumblebee/TODO:0.0.7
7588
`,
7689
Args: cobra.ExactArgs(1), // Filename or image
7790
RunE: func(cmd *cobra.Command, args []string) error {
@@ -118,6 +131,13 @@ func run(cmd *cobra.Command, args []string, opts *runOptions) error {
118131
return fmt.Errorf("could not parse BPF program: %w", err)
119132
}
120133

134+
watchMapOptions, err := parseWatchMapOptions(opts)
135+
if err != nil {
136+
contextutils.LoggerFrom(ctx).Errorf("could not parse watch map options: %v", err)
137+
return err
138+
}
139+
parsedELF.WatchedMapOptions = watchMapOptions
140+
121141
tuiApp, err := buildTuiApp(&progLoader, progLocation, opts.filter, parsedELF)
122142
if err != nil {
123143
return err
@@ -244,3 +264,58 @@ func buildContext(ctx context.Context, debug bool) (context.Context, error) {
244264

245265
return ctx, nil
246266
}
267+
268+
func parseWatchMapOptions(runOpts *runOptions) (map[string]loader.WatchedMapOptions, error) {
269+
watchMapOptions := make(map[string]loader.WatchedMapOptions)
270+
271+
for _, bucket := range runOpts.histBuckets {
272+
mapName, bucketLimits, err := parseBucket(bucket)
273+
if err != nil {
274+
return nil, err
275+
}
276+
watchMapOptions[mapName] = loader.WatchedMapOptions{
277+
HistBuckets: bucketLimits,
278+
}
279+
}
280+
281+
for _, key := range runOpts.histValueKey {
282+
split := strings.Index(key, ",")
283+
if split == -1 {
284+
return nil, fmt.Errorf("could not parse hist-value-key: %s", key)
285+
}
286+
mapName := key[:split]
287+
valueKey := key[split+1:]
288+
if _, ok := watchMapOptions[mapName]; !ok {
289+
watchMapOptions[mapName] = loader.WatchedMapOptions{}
290+
}
291+
w := watchMapOptions[mapName]
292+
w.HistValueKey = valueKey
293+
watchMapOptions[mapName] = w
294+
}
295+
296+
return watchMapOptions, nil
297+
}
298+
299+
func parseBucket(bucket string) (string, []float64, error) {
300+
split := strings.Index(bucket, ",")
301+
if split == -1 {
302+
return "", nil, fmt.Errorf("could not parse bucket: %s", bucket)
303+
}
304+
305+
mapName := bucket[:split]
306+
bucketLimits := bucket[split+1:]
307+
bucketLimits = strings.TrimPrefix(bucketLimits, "[")
308+
bucketLimits = strings.TrimSuffix(bucketLimits, "]")
309+
buckets := []float64{}
310+
311+
for _, limit := range strings.Split(bucketLimits, ",") {
312+
bval, err := strconv.ParseFloat(limit, 64)
313+
if err != nil {
314+
return "", nil, fmt.Errorf("could not parse bucket: %s from buckets %s", limit, bucket)
315+
}
316+
buckets = append(buckets, bval)
317+
}
318+
319+
return mapName, buckets, nil
320+
321+
}

‎pkg/decoder/decoder.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,9 @@ func (d *decoder) processSingleType(typ btf.Type) (interface{}, error) {
9292
case *btf.Int:
9393
switch typedMember.Encoding {
9494
case btf.Signed:
95+
if typedMember.Name == "char" {
96+
return d.handleChar(typedMember)
97+
}
9598
return d.handleInt(typedMember)
9699
case btf.Bool:
97100
// TODO
@@ -100,6 +103,9 @@ func (d *decoder) processSingleType(typ btf.Type) (interface{}, error) {
100103
// TODO
101104
return "", nil
102105
default:
106+
if typedMember.Name == "unsigned char" {
107+
return d.handleChar(typedMember)
108+
}
103109
// Default encoding seems to be unsigned
104110
return d.handleUint(typedMember)
105111
}
@@ -221,6 +227,18 @@ func (d *decoder) handleUint(
221227
return nil, errors.New("this should never happen")
222228
}
223229

230+
func (d *decoder) handleChar(
231+
typedMember *btf.Int,
232+
) (interface{}, error) {
233+
buf := bytes.NewBuffer(d.raw[d.offset : d.offset+1])
234+
d.offset += 1
235+
var val byte
236+
if err := binary.Read(buf, Endianess, &val); err != nil {
237+
return nil, err
238+
}
239+
return string([]byte{val}), nil
240+
}
241+
224242
func (d *decoder) handleInt(
225243
typedMember *btf.Int,
226244
) (interface{}, error) {

0 commit comments

Comments
 (0)