Skip to content

Commit 167fd21

Browse files
committed
x86/entry: Use generic syscall exit functionality
Replace the x86 variant with the generic version. Provide the relevant architecture specific helper functions and defines. Use a temporary define for idtentry_exit_user which will be cleaned up seperately. Signed-off-by: Thomas Gleixner <[email protected]> Acked-by: Kees Cook <[email protected]> Link: https://lkml.kernel.org/r/[email protected]
1 parent 27d6b4d commit 167fd21

File tree

7 files changed

+54
-222
lines changed

7 files changed

+54
-222
lines changed

arch/x86/entry/common.c

Lines changed: 5 additions & 216 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,8 @@
1515
#include <linux/smp.h>
1616
#include <linux/errno.h>
1717
#include <linux/ptrace.h>
18-
#include <linux/tracehook.h>
19-
#include <linux/audit.h>
20-
#include <linux/signal.h>
2118
#include <linux/export.h>
22-
#include <linux/context_tracking.h>
23-
#include <linux/user-return-notifier.h>
2419
#include <linux/nospec.h>
25-
#include <linux/uprobes.h>
26-
#include <linux/livepatch.h>
2720
#include <linux/syscalls.h>
2821
#include <linux/uaccess.h>
2922

@@ -42,191 +35,6 @@
4235
#include <asm/syscall.h>
4336
#include <asm/irq_stack.h>
4437

45-
#include <trace/events/syscalls.h>
46-
47-
/**
48-
* exit_to_user_mode - Fixup state when exiting to user mode
49-
*
50-
* Syscall exit enables interrupts, but the kernel state is interrupts
51-
* disabled when this is invoked. Also tell RCU about it.
52-
*
53-
* 1) Trace interrupts on state
54-
* 2) Invoke context tracking if enabled to adjust RCU state
55-
* 3) Clear CPU buffers if CPU is affected by MDS and the migitation is on.
56-
* 4) Tell lockdep that interrupts are enabled
57-
*/
58-
static __always_inline void exit_to_user_mode(void)
59-
{
60-
instrumentation_begin();
61-
trace_hardirqs_on_prepare();
62-
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
63-
instrumentation_end();
64-
65-
user_enter_irqoff();
66-
mds_user_clear_cpu_buffers();
67-
lockdep_hardirqs_on(CALLER_ADDR0);
68-
}
69-
70-
#define EXIT_TO_USERMODE_LOOP_FLAGS \
71-
(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
72-
_TIF_NEED_RESCHED | _TIF_PATCH_PENDING)
73-
74-
static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
75-
{
76-
/*
77-
* In order to return to user mode, we need to have IRQs off with
78-
* none of EXIT_TO_USERMODE_LOOP_FLAGS set. Several of these flags
79-
* can be set at any time on preemptible kernels if we have IRQs on,
80-
* so we need to loop. Disabling preemption wouldn't help: doing the
81-
* work to clear some of the flags can sleep.
82-
*/
83-
while (true) {
84-
/* We have work to do. */
85-
local_irq_enable();
86-
87-
if (cached_flags & _TIF_NEED_RESCHED)
88-
schedule();
89-
90-
if (cached_flags & _TIF_UPROBE)
91-
uprobe_notify_resume(regs);
92-
93-
if (cached_flags & _TIF_PATCH_PENDING)
94-
klp_update_patch_state(current);
95-
96-
/* deal with pending signal delivery */
97-
if (cached_flags & _TIF_SIGPENDING)
98-
do_signal(regs);
99-
100-
if (cached_flags & _TIF_NOTIFY_RESUME) {
101-
clear_thread_flag(TIF_NOTIFY_RESUME);
102-
tracehook_notify_resume(regs);
103-
rseq_handle_notify_resume(NULL, regs);
104-
}
105-
106-
/* Disable IRQs and retry */
107-
local_irq_disable();
108-
109-
cached_flags = READ_ONCE(current_thread_info()->flags);
110-
111-
if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
112-
break;
113-
}
114-
}
115-
116-
static void __prepare_exit_to_usermode(struct pt_regs *regs)
117-
{
118-
struct thread_info *ti = current_thread_info();
119-
u32 cached_flags;
120-
121-
addr_limit_user_check();
122-
123-
lockdep_assert_irqs_disabled();
124-
lockdep_sys_exit();
125-
126-
cached_flags = READ_ONCE(ti->flags);
127-
128-
if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
129-
exit_to_usermode_loop(regs, cached_flags);
130-
131-
/* Reload ti->flags; we may have rescheduled above. */
132-
cached_flags = READ_ONCE(ti->flags);
133-
134-
if (cached_flags & _TIF_USER_RETURN_NOTIFY)
135-
fire_user_return_notifiers();
136-
137-
if (unlikely(cached_flags & _TIF_IO_BITMAP))
138-
tss_update_io_bitmap();
139-
140-
fpregs_assert_state_consistent();
141-
if (unlikely(cached_flags & _TIF_NEED_FPU_LOAD))
142-
switch_fpu_return();
143-
144-
#ifdef CONFIG_COMPAT
145-
/*
146-
* Compat syscalls set TS_COMPAT. Make sure we clear it before
147-
* returning to user mode. We need to clear it *after* signal
148-
* handling, because syscall restart has a fixup for compat
149-
* syscalls. The fixup is exercised by the ptrace_syscall_32
150-
* selftest.
151-
*
152-
* We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
153-
* special case only applies after poking regs and before the
154-
* very next return to user mode.
155-
*/
156-
ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);
157-
#endif
158-
}
159-
160-
static noinstr void prepare_exit_to_usermode(struct pt_regs *regs)
161-
{
162-
instrumentation_begin();
163-
__prepare_exit_to_usermode(regs);
164-
instrumentation_end();
165-
exit_to_user_mode();
166-
}
167-
168-
#define SYSCALL_EXIT_WORK_FLAGS \
169-
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
170-
_TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)
171-
172-
static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)
173-
{
174-
bool step;
175-
176-
audit_syscall_exit(regs);
177-
178-
if (cached_flags & _TIF_SYSCALL_TRACEPOINT)
179-
trace_sys_exit(regs, regs->ax);
180-
181-
/*
182-
* If TIF_SYSCALL_EMU is set, we only get here because of
183-
* TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
184-
* We already reported this syscall instruction in
185-
* syscall_trace_enter().
186-
*/
187-
step = unlikely(
188-
(cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))
189-
== _TIF_SINGLESTEP);
190-
if (step || cached_flags & _TIF_SYSCALL_TRACE)
191-
tracehook_report_syscall_exit(regs, step);
192-
}
193-
194-
static void __syscall_return_slowpath(struct pt_regs *regs)
195-
{
196-
struct thread_info *ti = current_thread_info();
197-
u32 cached_flags = READ_ONCE(ti->flags);
198-
199-
CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
200-
201-
if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
202-
WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))
203-
local_irq_enable();
204-
205-
rseq_syscall(regs);
206-
207-
/*
208-
* First do one-time work. If these work items are enabled, we
209-
* want to run them exactly once per syscall exit with IRQs on.
210-
*/
211-
if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS))
212-
syscall_slow_exit_work(regs, cached_flags);
213-
214-
local_irq_disable();
215-
__prepare_exit_to_usermode(regs);
216-
}
217-
218-
/*
219-
* Called with IRQs on and fully valid regs. Returns with IRQs off in a
220-
* state such that we can immediately switch to user mode.
221-
*/
222-
__visible noinstr void syscall_return_slowpath(struct pt_regs *regs)
223-
{
224-
instrumentation_begin();
225-
__syscall_return_slowpath(regs);
226-
instrumentation_end();
227-
exit_to_user_mode();
228-
}
229-
23038
#ifdef CONFIG_X86_64
23139
__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
23240
{
@@ -245,7 +53,7 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
24553
#endif
24654
}
24755
instrumentation_end();
248-
syscall_return_slowpath(regs);
56+
syscall_exit_to_user_mode(regs);
24957
}
25058
#endif
25159

@@ -284,7 +92,7 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
28492
unsigned int nr = syscall_32_enter(regs);
28593

28694
do_syscall_32_irqs_on(regs, nr);
287-
syscall_return_slowpath(regs);
95+
syscall_exit_to_user_mode(regs);
28896
}
28997

29098
static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
@@ -310,13 +118,13 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
310118
if (res) {
311119
/* User code screwed up. */
312120
regs->ax = -EFAULT;
313-
syscall_return_slowpath(regs);
121+
syscall_exit_to_user_mode(regs);
314122
return false;
315123
}
316124

317125
/* Now this is just like a normal syscall. */
318126
do_syscall_32_irqs_on(regs, nr);
319-
syscall_return_slowpath(regs);
127+
syscall_exit_to_user_mode(regs);
320128
return true;
321129
}
322130

@@ -524,7 +332,7 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
524332

525333
/* Check whether this returns to user mode */
526334
if (user_mode(regs)) {
527-
prepare_exit_to_usermode(regs);
335+
irqentry_exit_to_user_mode(regs);
528336
} else if (regs->flags & X86_EFLAGS_IF) {
529337
/*
530338
* If RCU was not watching on entry this needs to be done
@@ -555,25 +363,6 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
555363
}
556364
}
557365

558-
/**
559-
* idtentry_exit_user - Handle return from exception to user mode
560-
* @regs: Pointer to pt_regs (exception entry regs)
561-
*
562-
* Runs the necessary preemption and work checks and returns to the caller
563-
* with interrupts disabled and no further work pending.
564-
*
565-
* This is the last action before returning to the low level ASM code which
566-
* just needs to return to the appropriate context.
567-
*
568-
* Counterpart to idtentry_enter_user().
569-
*/
570-
void noinstr idtentry_exit_user(struct pt_regs *regs)
571-
{
572-
lockdep_assert_irqs_disabled();
573-
574-
prepare_exit_to_usermode(regs);
575-
}
576-
577366
#ifdef CONFIG_XEN_PV
578367
#ifndef CONFIG_PREEMPTION
579368
/*

arch/x86/entry/entry_32.S

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -846,7 +846,7 @@ SYM_CODE_START(ret_from_fork)
846846
2:
847847
/* When we fork, we trace the syscall return in the child, too. */
848848
movl %esp, %eax
849-
call syscall_return_slowpath
849+
call syscall_exit_to_user_mode
850850
jmp .Lsyscall_32_done
851851

852852
/* kernel thread */

arch/x86/entry/entry_64.S

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ SYM_CODE_START(ret_from_fork)
283283
2:
284284
UNWIND_HINT_REGS
285285
movq %rsp, %rdi
286-
call syscall_return_slowpath /* returns with IRQs disabled */
286+
call syscall_exit_to_user_mode /* returns with IRQs disabled */
287287
jmp swapgs_restore_regs_and_return_to_usermode
288288

289289
1:

arch/x86/include/asm/entry-common.h

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
#ifndef _ASM_X86_ENTRY_COMMON_H
33
#define _ASM_X86_ENTRY_COMMON_H
44

5+
#include <linux/user-return-notifier.h>
6+
7+
#include <asm/nospec-branch.h>
8+
#include <asm/io_bitmap.h>
9+
#include <asm/fpu/api.h>
10+
511
/* Check that the stack and regs on entry from user mode are sane. */
612
static __always_inline void arch_check_user_regs(struct pt_regs *regs)
713
{
@@ -29,4 +35,42 @@ static __always_inline void arch_check_user_regs(struct pt_regs *regs)
2935
}
3036
#define arch_check_user_regs arch_check_user_regs
3137

38+
#define ARCH_SYSCALL_EXIT_WORK (_TIF_SINGLESTEP)
39+
40+
static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
41+
unsigned long ti_work)
42+
{
43+
if (ti_work & _TIF_USER_RETURN_NOTIFY)
44+
fire_user_return_notifiers();
45+
46+
if (unlikely(ti_work & _TIF_IO_BITMAP))
47+
tss_update_io_bitmap();
48+
49+
fpregs_assert_state_consistent();
50+
if (unlikely(ti_work & _TIF_NEED_FPU_LOAD))
51+
switch_fpu_return();
52+
53+
#ifdef CONFIG_COMPAT
54+
/*
55+
* Compat syscalls set TS_COMPAT. Make sure we clear it before
56+
* returning to user mode. We need to clear it *after* signal
57+
* handling, because syscall restart has a fixup for compat
58+
* syscalls. The fixup is exercised by the ptrace_syscall_32
59+
* selftest.
60+
*
61+
* We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
62+
* special case only applies after poking regs and before the
63+
* very next return to user mode.
64+
*/
65+
current_thread_info()->status &= ~(TS_COMPAT | TS_I386_REGS_POKED);
66+
#endif
67+
}
68+
#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
69+
70+
static __always_inline void arch_exit_to_user_mode(void)
71+
{
72+
mds_user_clear_cpu_buffers();
73+
}
74+
#define arch_exit_to_user_mode arch_exit_to_user_mode
75+
3276
#endif

arch/x86/include/asm/idtentry.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,7 @@
1313

1414
/* Temporary define */
1515
#define idtentry_enter_user irqentry_enter_from_user_mode
16-
17-
void idtentry_exit_user(struct pt_regs *regs);
16+
#define idtentry_exit_user irqentry_exit_to_user_mode
1817

1918
typedef struct idtentry_state {
2019
bool exit_rcu;

arch/x86/include/asm/signal.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ typedef sigset_t compat_sigset_t;
3535
#endif /* __ASSEMBLY__ */
3636
#include <uapi/asm/signal.h>
3737
#ifndef __ASSEMBLY__
38-
extern void do_signal(struct pt_regs *regs);
3938

4039
#define __ARCH_HAS_SA_RESTORER
4140

arch/x86/kernel/signal.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <linux/user-return-notifier.h>
2626
#include <linux/uprobes.h>
2727
#include <linux/context_tracking.h>
28+
#include <linux/entry-common.h>
2829
#include <linux/syscalls.h>
2930

3031
#include <asm/processor.h>
@@ -803,7 +804,7 @@ static inline unsigned long get_nr_restart_syscall(const struct pt_regs *regs)
803804
* want to handle. Thus you cannot kill init even with a SIGKILL even by
804805
* mistake.
805806
*/
806-
void do_signal(struct pt_regs *regs)
807+
void arch_do_signal(struct pt_regs *regs)
807808
{
808809
struct ksignal ksig;
809810

0 commit comments

Comments
 (0)