15
15
#include <linux/smp.h>
16
16
#include <linux/errno.h>
17
17
#include <linux/ptrace.h>
18
- #include <linux/tracehook.h>
19
- #include <linux/audit.h>
20
- #include <linux/signal.h>
21
18
#include <linux/export.h>
22
- #include <linux/context_tracking.h>
23
- #include <linux/user-return-notifier.h>
24
19
#include <linux/nospec.h>
25
- #include <linux/uprobes.h>
26
- #include <linux/livepatch.h>
27
20
#include <linux/syscalls.h>
28
21
#include <linux/uaccess.h>
29
22
42
35
#include <asm/syscall.h>
43
36
#include <asm/irq_stack.h>
44
37
45
- #include <trace/events/syscalls.h>
46
-
47
- /**
48
- * exit_to_user_mode - Fixup state when exiting to user mode
49
- *
50
- * Syscall exit enables interrupts, but the kernel state is interrupts
51
- * disabled when this is invoked. Also tell RCU about it.
52
- *
53
- * 1) Trace interrupts on state
54
- * 2) Invoke context tracking if enabled to adjust RCU state
55
- * 3) Clear CPU buffers if CPU is affected by MDS and the migitation is on.
56
- * 4) Tell lockdep that interrupts are enabled
57
- */
58
- static __always_inline void exit_to_user_mode (void )
59
- {
60
- instrumentation_begin ();
61
- trace_hardirqs_on_prepare ();
62
- lockdep_hardirqs_on_prepare (CALLER_ADDR0 );
63
- instrumentation_end ();
64
-
65
- user_enter_irqoff ();
66
- mds_user_clear_cpu_buffers ();
67
- lockdep_hardirqs_on (CALLER_ADDR0 );
68
- }
69
-
70
- #define EXIT_TO_USERMODE_LOOP_FLAGS \
71
- (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
72
- _TIF_NEED_RESCHED | _TIF_PATCH_PENDING)
73
-
74
- static void exit_to_usermode_loop (struct pt_regs * regs , u32 cached_flags )
75
- {
76
- /*
77
- * In order to return to user mode, we need to have IRQs off with
78
- * none of EXIT_TO_USERMODE_LOOP_FLAGS set. Several of these flags
79
- * can be set at any time on preemptible kernels if we have IRQs on,
80
- * so we need to loop. Disabling preemption wouldn't help: doing the
81
- * work to clear some of the flags can sleep.
82
- */
83
- while (true) {
84
- /* We have work to do. */
85
- local_irq_enable ();
86
-
87
- if (cached_flags & _TIF_NEED_RESCHED )
88
- schedule ();
89
-
90
- if (cached_flags & _TIF_UPROBE )
91
- uprobe_notify_resume (regs );
92
-
93
- if (cached_flags & _TIF_PATCH_PENDING )
94
- klp_update_patch_state (current );
95
-
96
- /* deal with pending signal delivery */
97
- if (cached_flags & _TIF_SIGPENDING )
98
- do_signal (regs );
99
-
100
- if (cached_flags & _TIF_NOTIFY_RESUME ) {
101
- clear_thread_flag (TIF_NOTIFY_RESUME );
102
- tracehook_notify_resume (regs );
103
- rseq_handle_notify_resume (NULL , regs );
104
- }
105
-
106
- /* Disable IRQs and retry */
107
- local_irq_disable ();
108
-
109
- cached_flags = READ_ONCE (current_thread_info ()-> flags );
110
-
111
- if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS ))
112
- break ;
113
- }
114
- }
115
-
116
- static void __prepare_exit_to_usermode (struct pt_regs * regs )
117
- {
118
- struct thread_info * ti = current_thread_info ();
119
- u32 cached_flags ;
120
-
121
- addr_limit_user_check ();
122
-
123
- lockdep_assert_irqs_disabled ();
124
- lockdep_sys_exit ();
125
-
126
- cached_flags = READ_ONCE (ti -> flags );
127
-
128
- if (unlikely (cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS ))
129
- exit_to_usermode_loop (regs , cached_flags );
130
-
131
- /* Reload ti->flags; we may have rescheduled above. */
132
- cached_flags = READ_ONCE (ti -> flags );
133
-
134
- if (cached_flags & _TIF_USER_RETURN_NOTIFY )
135
- fire_user_return_notifiers ();
136
-
137
- if (unlikely (cached_flags & _TIF_IO_BITMAP ))
138
- tss_update_io_bitmap ();
139
-
140
- fpregs_assert_state_consistent ();
141
- if (unlikely (cached_flags & _TIF_NEED_FPU_LOAD ))
142
- switch_fpu_return ();
143
-
144
- #ifdef CONFIG_COMPAT
145
- /*
146
- * Compat syscalls set TS_COMPAT. Make sure we clear it before
147
- * returning to user mode. We need to clear it *after* signal
148
- * handling, because syscall restart has a fixup for compat
149
- * syscalls. The fixup is exercised by the ptrace_syscall_32
150
- * selftest.
151
- *
152
- * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
153
- * special case only applies after poking regs and before the
154
- * very next return to user mode.
155
- */
156
- ti -> status &= ~(TS_COMPAT |TS_I386_REGS_POKED );
157
- #endif
158
- }
159
-
160
- static noinstr void prepare_exit_to_usermode (struct pt_regs * regs )
161
- {
162
- instrumentation_begin ();
163
- __prepare_exit_to_usermode (regs );
164
- instrumentation_end ();
165
- exit_to_user_mode ();
166
- }
167
-
168
- #define SYSCALL_EXIT_WORK_FLAGS \
169
- (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
170
- _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)
171
-
172
- static void syscall_slow_exit_work (struct pt_regs * regs , u32 cached_flags )
173
- {
174
- bool step ;
175
-
176
- audit_syscall_exit (regs );
177
-
178
- if (cached_flags & _TIF_SYSCALL_TRACEPOINT )
179
- trace_sys_exit (regs , regs -> ax );
180
-
181
- /*
182
- * If TIF_SYSCALL_EMU is set, we only get here because of
183
- * TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
184
- * We already reported this syscall instruction in
185
- * syscall_trace_enter().
186
- */
187
- step = unlikely (
188
- (cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU ))
189
- == _TIF_SINGLESTEP );
190
- if (step || cached_flags & _TIF_SYSCALL_TRACE )
191
- tracehook_report_syscall_exit (regs , step );
192
- }
193
-
194
- static void __syscall_return_slowpath (struct pt_regs * regs )
195
- {
196
- struct thread_info * ti = current_thread_info ();
197
- u32 cached_flags = READ_ONCE (ti -> flags );
198
-
199
- CT_WARN_ON (ct_state () != CONTEXT_KERNEL );
200
-
201
- if (IS_ENABLED (CONFIG_PROVE_LOCKING ) &&
202
- WARN (irqs_disabled (), "syscall %ld left IRQs disabled" , regs -> orig_ax ))
203
- local_irq_enable ();
204
-
205
- rseq_syscall (regs );
206
-
207
- /*
208
- * First do one-time work. If these work items are enabled, we
209
- * want to run them exactly once per syscall exit with IRQs on.
210
- */
211
- if (unlikely (cached_flags & SYSCALL_EXIT_WORK_FLAGS ))
212
- syscall_slow_exit_work (regs , cached_flags );
213
-
214
- local_irq_disable ();
215
- __prepare_exit_to_usermode (regs );
216
- }
217
-
218
- /*
219
- * Called with IRQs on and fully valid regs. Returns with IRQs off in a
220
- * state such that we can immediately switch to user mode.
221
- */
222
- __visible noinstr void syscall_return_slowpath (struct pt_regs * regs )
223
- {
224
- instrumentation_begin ();
225
- __syscall_return_slowpath (regs );
226
- instrumentation_end ();
227
- exit_to_user_mode ();
228
- }
229
-
230
38
#ifdef CONFIG_X86_64
231
39
__visible noinstr void do_syscall_64 (unsigned long nr , struct pt_regs * regs )
232
40
{
@@ -245,7 +53,7 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
245
53
#endif
246
54
}
247
55
instrumentation_end ();
248
- syscall_return_slowpath (regs );
56
+ syscall_exit_to_user_mode (regs );
249
57
}
250
58
#endif
251
59
@@ -284,7 +92,7 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
284
92
unsigned int nr = syscall_32_enter (regs );
285
93
286
94
do_syscall_32_irqs_on (regs , nr );
287
- syscall_return_slowpath (regs );
95
+ syscall_exit_to_user_mode (regs );
288
96
}
289
97
290
98
static noinstr bool __do_fast_syscall_32 (struct pt_regs * regs )
@@ -310,13 +118,13 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
310
118
if (res ) {
311
119
/* User code screwed up. */
312
120
regs -> ax = - EFAULT ;
313
- syscall_return_slowpath (regs );
121
+ syscall_exit_to_user_mode (regs );
314
122
return false;
315
123
}
316
124
317
125
/* Now this is just like a normal syscall. */
318
126
do_syscall_32_irqs_on (regs , nr );
319
- syscall_return_slowpath (regs );
127
+ syscall_exit_to_user_mode (regs );
320
128
return true;
321
129
}
322
130
@@ -524,7 +332,7 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
524
332
525
333
/* Check whether this returns to user mode */
526
334
if (user_mode (regs )) {
527
- prepare_exit_to_usermode (regs );
335
+ irqentry_exit_to_user_mode (regs );
528
336
} else if (regs -> flags & X86_EFLAGS_IF ) {
529
337
/*
530
338
* If RCU was not watching on entry this needs to be done
@@ -555,25 +363,6 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
555
363
}
556
364
}
557
365
558
- /**
559
- * idtentry_exit_user - Handle return from exception to user mode
560
- * @regs: Pointer to pt_regs (exception entry regs)
561
- *
562
- * Runs the necessary preemption and work checks and returns to the caller
563
- * with interrupts disabled and no further work pending.
564
- *
565
- * This is the last action before returning to the low level ASM code which
566
- * just needs to return to the appropriate context.
567
- *
568
- * Counterpart to idtentry_enter_user().
569
- */
570
- void noinstr idtentry_exit_user (struct pt_regs * regs )
571
- {
572
- lockdep_assert_irqs_disabled ();
573
-
574
- prepare_exit_to_usermode (regs );
575
- }
576
-
577
366
#ifdef CONFIG_XEN_PV
578
367
#ifndef CONFIG_PREEMPTION
579
368
/*
0 commit comments