Skip to content

Commit 6ea082b

Browse files
lenticularis39rostedt
lenticularis39
authored and
rostedt
committed
rtla/timerlat: Add action on threshold feature
Extend the functionality provided by the -t/--trace option, which triggers saving the contents of a tracefs buffer after tracing is stopped, to support implementing arbitrary actions. A new option, --on-threshold, is added, taking an argument that further specifies the action. Actions added in this patch are: - trace[,file=<filename>]: Saves tracefs buffer, optionally taking a filename. - signal,num=<sig>,pid=<pid>: Sends signal to process. "parent" might be specified instead of number to send signal to parent process. - shell,command=<command>: Execute shell command. Multiple actions may be specified and will be executed in order, including multiple actions of the same type. Trace output requested via -t and -a now adds a trace action to the end of the list. If an action fails, the following actions are not executed. For example, this command: $ rtla timerlat -T 20 --on-threshold trace \ --on-threshold shell,command="grep ipi_send timerlat_trace.txt" \ --on-threshold signal,num=2,pid=parent will send signal 2 (SIGINT) to parent process, but only if saved trace contains the text "ipi_send". This way, the feature can be used for flexible reactions on latency spikes, and allows combining rtla with other tooling like perf. Cc: John Kacur <[email protected]> Cc: Luis Goncalves <[email protected]> Cc: Arnaldo Carvalho de Melo <[email protected]> Cc: Chang Yin <[email protected]> Cc: Costa Shulyupin <[email protected]> Cc: Crystal Wood <[email protected]> Cc: Gabriele Monaco <[email protected]> Link: https://lore.kernel.org/[email protected] Signed-off-by: Tomas Glozar <[email protected]> Signed-off-by: Steven Rostedt (Google) <[email protected]>
1 parent 8b6cbca commit 6ea082b

File tree

6 files changed

+341
-22
lines changed

6 files changed

+341
-22
lines changed

‎tools/tracing/rtla/src/Build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
rtla-y += trace.o
22
rtla-y += utils.o
3+
rtla-y += actions.o
34
rtla-y += osnoise.o
45
rtla-y += osnoise_top.o
56
rtla-y += osnoise_hist.o

‎tools/tracing/rtla/src/actions.c

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
#include <stdlib.h>
3+
#include <string.h>
4+
#include <signal.h>
5+
#include <unistd.h>
6+
7+
#include "actions.h"
8+
#include "trace.h"
9+
#include "utils.h"
10+
11+
/*
12+
* actions_init - initialize struct actions
13+
*/
14+
void
15+
actions_init(struct actions *self)
16+
{
17+
self->size = action_default_size;
18+
self->list = calloc(self->size, sizeof(struct action));
19+
self->len = 0;
20+
21+
memset(&self->present, 0, sizeof(self->present));
22+
23+
/* This has to be set by the user */
24+
self->trace_output_inst = NULL;
25+
}
26+
27+
/*
28+
* actions_destroy - destroy struct actions
29+
*/
30+
void
31+
actions_destroy(struct actions *self)
32+
{
33+
/* Free any action-specific data */
34+
for (struct action *action = self->list; action < self->list + self->len; action++) {
35+
if (action->type == ACTION_SHELL)
36+
free(action->command);
37+
if (action->type == ACTION_TRACE_OUTPUT)
38+
free(action->trace_output);
39+
}
40+
41+
/* Free action list */
42+
free(self->list);
43+
}
44+
45+
/*
46+
* actions_new - Get pointer to new action
47+
*/
48+
static struct action *
49+
actions_new(struct actions *self)
50+
{
51+
if (self->size >= self->len) {
52+
self->size *= 2;
53+
self->list = realloc(self->list, self->size * sizeof(struct action));
54+
}
55+
56+
return &self->list[self->len++];
57+
}
58+
59+
/*
60+
* actions_add_trace_output - add an action to output trace
61+
*/
62+
int
63+
actions_add_trace_output(struct actions *self, const char *trace_output)
64+
{
65+
struct action *action = actions_new(self);
66+
67+
self->present[ACTION_TRACE_OUTPUT] = true;
68+
action->type = ACTION_TRACE_OUTPUT;
69+
action->trace_output = calloc(strlen(trace_output) + 1, sizeof(char));
70+
if (!action->trace_output)
71+
return -1;
72+
strcpy(action->trace_output, trace_output);
73+
74+
return 0;
75+
}
76+
77+
/*
78+
* actions_add_trace_output - add an action to send signal to a process
79+
*/
80+
int
81+
actions_add_signal(struct actions *self, int signal, int pid)
82+
{
83+
struct action *action = actions_new(self);
84+
85+
self->present[ACTION_SIGNAL] = true;
86+
action->type = ACTION_SIGNAL;
87+
action->signal = signal;
88+
action->pid = pid;
89+
90+
return 0;
91+
}
92+
93+
/*
94+
* actions_add_shell - add an action to execute a shell command
95+
*/
96+
int
97+
actions_add_shell(struct actions *self, const char *command)
98+
{
99+
struct action *action = actions_new(self);
100+
101+
self->present[ACTION_SHELL] = true;
102+
action->type = ACTION_SHELL;
103+
action->command = calloc(strlen(command) + 1, sizeof(char));
104+
if (!action->command)
105+
return -1;
106+
strcpy(action->command, command);
107+
108+
return 0;
109+
}
110+
111+
/*
112+
* actions_parse - add an action based on text specification
113+
*/
114+
int
115+
actions_parse(struct actions *self, const char *trigger)
116+
{
117+
enum action_type type = ACTION_NONE;
118+
char *token;
119+
char trigger_c[strlen(trigger)];
120+
121+
/* For ACTION_SIGNAL */
122+
int signal = 0, pid = 0;
123+
124+
/* For ACTION_TRACE_OUTPUT */
125+
char *trace_output;
126+
127+
strcpy(trigger_c, trigger);
128+
token = strtok(trigger_c, ",");
129+
130+
if (strcmp(token, "trace") == 0)
131+
type = ACTION_TRACE_OUTPUT;
132+
else if (strcmp(token, "signal") == 0)
133+
type = ACTION_SIGNAL;
134+
else if (strcmp(token, "shell") == 0)
135+
type = ACTION_SHELL;
136+
else
137+
/* Invalid trigger type */
138+
return -1;
139+
140+
token = strtok(NULL, ",");
141+
142+
switch (type) {
143+
case ACTION_TRACE_OUTPUT:
144+
/* Takes no argument */
145+
if (token == NULL)
146+
trace_output = "timerlat_trace.txt";
147+
else {
148+
if (strlen(token) > 5 && strncmp(token, "file=", 5) == 0) {
149+
trace_output = token + 5;
150+
} else {
151+
/* Invalid argument */
152+
return -1;
153+
}
154+
155+
token = strtok(NULL, ",");
156+
if (token != NULL)
157+
/* Only one argument allowed */
158+
return -1;
159+
}
160+
return actions_add_trace_output(self, trace_output);
161+
case ACTION_SIGNAL:
162+
/* Takes two arguments, num (signal) and pid */
163+
while (token != NULL) {
164+
if (strlen(token) > 4 && strncmp(token, "num=", 4) == 0) {
165+
signal = atoi(token + 4);
166+
} else if (strlen(token) > 4 && strncmp(token, "pid=", 4) == 0) {
167+
if (strncmp(token + 4, "parent", 7) == 0)
168+
pid = -1;
169+
else
170+
pid = atoi(token + 4);
171+
} else {
172+
/* Invalid argument */
173+
return -1;
174+
}
175+
176+
token = strtok(NULL, ",");
177+
}
178+
179+
if (!signal || !pid)
180+
/* Missing argument */
181+
return -1;
182+
183+
return actions_add_signal(self, signal, pid);
184+
case ACTION_SHELL:
185+
if (token == NULL)
186+
return -1;
187+
if (strlen(token) > 8 && strncmp(token, "command=", 8) == 0)
188+
return actions_add_shell(self, token + 8);
189+
return -1;
190+
default:
191+
return -1;
192+
}
193+
}
194+
195+
/*
196+
* actions_perform - perform all actions
197+
*/
198+
int
199+
actions_perform(const struct actions *self)
200+
{
201+
int pid, retval;
202+
const struct action *action;
203+
204+
for (action = self->list; action < self->list + self->len; action++) {
205+
switch (action->type) {
206+
case ACTION_TRACE_OUTPUT:
207+
retval = save_trace_to_file(self->trace_output_inst, action->trace_output);
208+
if (retval) {
209+
err_msg("Error saving trace\n");
210+
return retval;
211+
}
212+
break;
213+
case ACTION_SIGNAL:
214+
if (action->pid == -1)
215+
pid = getppid();
216+
else
217+
pid = action->pid;
218+
retval = kill(pid, action->signal);
219+
if (retval) {
220+
err_msg("Error sending signal\n");
221+
return retval;
222+
}
223+
break;
224+
case ACTION_SHELL:
225+
retval = system(action->command);
226+
if (retval)
227+
return retval;
228+
break;
229+
default:
230+
break;
231+
}
232+
}
233+
234+
return 0;
235+
}

‎tools/tracing/rtla/src/actions.h

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
#include <tracefs.h>
3+
#include <stdbool.h>
4+
5+
enum action_type {
6+
ACTION_NONE = 0,
7+
ACTION_TRACE_OUTPUT,
8+
ACTION_SIGNAL,
9+
ACTION_SHELL,
10+
ACTION_FIELD_N
11+
};
12+
13+
struct action {
14+
enum action_type type;
15+
union {
16+
struct {
17+
/* For ACTION_TRACE_OUTPUT */
18+
char *trace_output;
19+
};
20+
struct {
21+
/* For ACTION_SIGNAL */
22+
int signal;
23+
int pid;
24+
};
25+
struct {
26+
/* For ACTION_SHELL */
27+
char *command;
28+
};
29+
};
30+
};
31+
32+
static const int action_default_size = 8;
33+
34+
struct actions {
35+
struct action *list;
36+
int len, size;
37+
bool present[ACTION_FIELD_N];
38+
39+
/* External dependencies */
40+
struct tracefs_instance *trace_output_inst;
41+
};
42+
43+
void actions_init(struct actions *self);
44+
void actions_destroy(struct actions *self);
45+
int actions_add_trace_output(struct actions *self, const char *trace_output);
46+
int actions_add_signal(struct actions *self, int signal, int pid);
47+
int actions_add_shell(struct actions *self, const char *command);
48+
int actions_parse(struct actions *self, const char *trigger);
49+
int actions_perform(const struct actions *self);

‎tools/tracing/rtla/src/timerlat.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// SPDX-License-Identifier: GPL-2.0
2+
#include "actions.h"
23
#include "osnoise.h"
34

45
/*
@@ -22,7 +23,6 @@ struct timerlat_params {
2223
/* Common params */
2324
char *cpus;
2425
cpu_set_t monitored_cpus;
25-
char *trace_output;
2626
char *cgroup_name;
2727
unsigned long long runtime;
2828
long long stop_us;
@@ -48,6 +48,7 @@ struct timerlat_params {
4848
struct sched_attr sched_param;
4949
struct trace_events *events;
5050
enum timerlat_tracing_mode mode;
51+
struct actions actions;
5152
union {
5253
struct {
5354
/* top only */

0 commit comments

Comments
 (0)