arm64: entry: call exit_to_user_mode() from C

mrutland-arm · ctmarinas · commit e130338eed5d · 2021-08-05T14:10:32.000+01:00
When handling an exception from EL0, we perform the entry work in that exception's C handler, and once the C handler has finished, we return back to the entry assembly. Subsequently in the common `ret_to_user` assembly we perform the exit work that balances with the entry work. This can be somewhat difficult to follow, and makes it hard to rework the return paths (e.g. to pass additional context to the exit code, or to have exception return logic for specific exceptions). This patch reworks the entry code such that each EL0 C exception handler is responsible for both the entry and exit work. This clearly balances the two (and will permit additional variation in future), and avoids an unnecessary bounce between assembly and C in the common case, leaving `ret_from_fork` as the only place assembly has to call the exit code. This means that the exit work is now inlined into the C handler, which is already the case for the entry work, and allows the compiler to generate better code (e.g. by immediately returning when there is no exit work to perform). To align with other exception entry/exit helpers, enter_from_user_mode() is updated to take the EL0 pt_regs as a parameter, though this is currently unused. There should be no functional change as a result of this patch. However, this should lead to slightly better backtraces when an error is encountered within do_notify_resume(), as the C handler should appear in the backtrace, indicating the specific exception that the kernel was entered with. Signed-off-by: Mark Rutland <mark.rutland@arm.com> Cc: James Morse <james.morse@arm.com> Cc: Joey Gouly <joey.gouly@arm.com> Cc: Marc Zyngier <maz@kernel.org> Cc: Will Deacon <will@kernel.org> Reviewed-by: Joey Gouly <joey.gouly@arm.com> Link: https://lore.kernel.org/r/20210802140733.52716-5-mark.rutland@arm.com Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
@@ -104,7 +104,7 @@ static __always_inline void __enter_from_user_mode(void)
 	trace_hardirqs_off_finish();
 }
 
-static __always_inline void enter_from_user_mode(void)
+static __always_inline void enter_from_user_mode(struct pt_regs *regs)
 {
 	__enter_from_user_mode();
 }
@@ -116,19 +116,12 @@ static __always_inline void enter_from_user_mode(void)
  */
 static __always_inline void __exit_to_user_mode(void)
 {
-
 	trace_hardirqs_on_prepare();
 	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
 	user_enter_irqoff();
 	lockdep_hardirqs_on(CALLER_ADDR0);
 }
 
-static __always_inline void exit_to_user_mode(void)
-{
-	mte_check_tfsr_exit();
-	__exit_to_user_mode();
-}
-
 static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs)
 {
 	unsigned long flags;
@@ -140,10 +133,16 @@ static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs)
 		do_notify_resume(regs, flags);
 }
 
-asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
+static __always_inline void exit_to_user_mode(struct pt_regs *regs)
 {
 	prepare_exit_to_user_mode(regs);
-	exit_to_user_mode();
+	mte_check_tfsr_exit();
+	__exit_to_user_mode();
+}
+
+asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
+{
+	exit_to_user_mode(regs);
 }
 
 /*
@@ -477,9 +476,10 @@ static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
 {
 	unsigned long far = read_sysreg(far_el1);
 
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_mem_abort(far, esr, regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
@@ -494,37 +494,42 @@ static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
 	if (!is_ttbr0_addr(far))
 		arm64_apply_bp_hardening();
 
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_mem_abort(far, esr, regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_fpsimd_acc(esr, regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_sve_acc(esr, regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_fpsimd_exc(esr, regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_sysinstr(esr, regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
@@ -534,61 +539,69 @@ static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
 	if (!is_ttbr0_addr(instruction_pointer(regs)))
 		arm64_apply_bp_hardening();
 
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_sp_pc_abort(far, esr, regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_sp_pc_abort(regs->sp, esr, regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_undef(struct pt_regs *regs)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_undefinstr(regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_bti(struct pt_regs *regs)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_bti(regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	bad_el0_sync(regs, 0, esr);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
 {
 	/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
 	unsigned long far = read_sysreg(far_el1);
 
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	do_debug_exception(far, esr, regs);
 	local_daif_restore(DAIF_PROCCTX);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_svc(struct pt_regs *regs)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	cortex_a76_erratum_1463225_svc_handler();
 	do_el0_svc(regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_ptrauth_fault(regs, esr);
+	exit_to_user_mode(regs);
 }
 
 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
@@ -647,14 +660,16 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
 static void noinstr el0_interrupt(struct pt_regs *regs,
 				  void (*handler)(struct pt_regs *))
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 
 	write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
 
 	if (regs->pc & BIT(55))
 		arm64_apply_bp_hardening();
 
 	do_interrupt_handler(regs, handler);
+
+	exit_to_user_mode(regs);
 }
 
 static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
@@ -681,12 +696,13 @@ static void noinstr __el0_error_handler_common(struct pt_regs *regs)
 {
 	unsigned long esr = read_sysreg(esr_el1);
 
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_ERRCTX);
 	arm64_enter_nmi(regs);
 	do_serror(regs, esr);
 	arm64_exit_nmi(regs);
 	local_daif_restore(DAIF_PROCCTX);
+	exit_to_user_mode(regs);
 }
 
 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
@@ -697,16 +713,18 @@ asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
 #ifdef CONFIG_COMPAT
 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
 	do_cp15instr(esr, regs);
+	exit_to_user_mode(regs);
 }
 
 static void noinstr el0_svc_compat(struct pt_regs *regs)
 {
-	enter_from_user_mode();
+	enter_from_user_mode(regs);
 	cortex_a76_erratum_1463225_svc_handler();
 	do_el0_svc_compat(regs);
+	exit_to_user_mode(regs);
 }
 
 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
@@ -564,8 +564,6 @@ SYM_CODE_START_LOCAL(ret_to_kernel)
 SYM_CODE_END(ret_to_kernel)
 
 SYM_CODE_START_LOCAL(ret_to_user)
-	mov	x0, sp
-	bl	asm_exit_to_user_mode
 	/* Ignore asynchronous tag check faults in the uaccess routines */
 	clear_mte_async_tcf
 	ldr	x19, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
@@ -739,6 +737,8 @@ SYM_CODE_START(ret_from_fork)
 	mov	x0, x20
 	blr	x19
 1:	get_current_task tsk
+	mov	x0, sp
+	bl	asm_exit_to_user_mode
 	b	ret_to_user
 SYM_CODE_END(ret_from_fork)
 NOKPROBE(ret_from_fork)

Original file line number	Diff line number	Diff line change
`@@ -104,7 +104,7 @@ static __always_inline void __enter_from_user_mode(void)`
`104`	`104`	`trace_hardirqs_off_finish();`
`105`	`105`	`}`
`106`	`106`
`107`		`-static __always_inline void enter_from_user_mode(void)`
	`107`	`+static __always_inline void enter_from_user_mode(struct pt_regs *regs)`
`108`	`108`	`{`
`109`	`109`	`__enter_from_user_mode();`
`110`	`110`	`}`
`@@ -116,19 +116,12 @@ static __always_inline void enter_from_user_mode(void)`
`116`	`116`	`*/`
`117`	`117`	`static __always_inline void __exit_to_user_mode(void)`
`118`	`118`	`{`
`119`		`-`
`120`	`119`	`trace_hardirqs_on_prepare();`
`121`	`120`	`lockdep_hardirqs_on_prepare(CALLER_ADDR0);`
`122`	`121`	`user_enter_irqoff();`
`123`	`122`	`lockdep_hardirqs_on(CALLER_ADDR0);`
`124`	`123`	`}`
`125`	`124`
`126`		`-static __always_inline void exit_to_user_mode(void)`
`127`		`-{`
`128`		`- mte_check_tfsr_exit();`
`129`		`- __exit_to_user_mode();`
`130`		`-}`
`131`		`-`
`132`	`125`	`static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs)`
`133`	`126`	`{`
`134`	`127`	`unsigned long flags;`
`@@ -140,10 +133,16 @@ static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs)`
`140`	`133`	`do_notify_resume(regs, flags);`
`141`	`134`	`}`
`142`	`135`
`143`		`-asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)`
	`136`	`+static __always_inline void exit_to_user_mode(struct pt_regs *regs)`
`144`	`137`	`{`
`145`	`138`	`prepare_exit_to_user_mode(regs);`
`146`		`- exit_to_user_mode();`
	`139`	`+ mte_check_tfsr_exit();`
	`140`	`+ __exit_to_user_mode();`
	`141`	`+}`
	`142`	`+`
	`143`	`+asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)`
	`144`	`+{`
	`145`	`+ exit_to_user_mode(regs);`
`147`	`146`	`}`
`148`	`147`
`149`	`148`	`/*`
`@@ -477,9 +476,10 @@ static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)`
`477`	`476`	`{`
`478`	`477`	`unsigned long far = read_sysreg(far_el1);`
`479`	`478`
`480`		`- enter_from_user_mode();`
	`479`	`+ enter_from_user_mode(regs);`
`481`	`480`	`local_daif_restore(DAIF_PROCCTX);`
`482`	`481`	`do_mem_abort(far, esr, regs);`
	`482`	`+ exit_to_user_mode(regs);`
`483`	`483`	`}`
`484`	`484`
`485`	`485`	`static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)`
`@@ -494,37 +494,42 @@ static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)`
`494`	`494`	`if (!is_ttbr0_addr(far))`
`495`	`495`	`arm64_apply_bp_hardening();`
`496`	`496`
`497`		`- enter_from_user_mode();`
	`497`	`+ enter_from_user_mode(regs);`
`498`	`498`	`local_daif_restore(DAIF_PROCCTX);`
`499`	`499`	`do_mem_abort(far, esr, regs);`
	`500`	`+ exit_to_user_mode(regs);`
`500`	`501`	`}`
`501`	`502`
`502`	`503`	`static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)`
`503`	`504`	`{`
`504`		`- enter_from_user_mode();`
	`505`	`+ enter_from_user_mode(regs);`
`505`	`506`	`local_daif_restore(DAIF_PROCCTX);`
`506`	`507`	`do_fpsimd_acc(esr, regs);`
	`508`	`+ exit_to_user_mode(regs);`
`507`	`509`	`}`
`508`	`510`
`509`	`511`	`static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)`
`510`	`512`	`{`
`511`		`- enter_from_user_mode();`
	`513`	`+ enter_from_user_mode(regs);`
`512`	`514`	`local_daif_restore(DAIF_PROCCTX);`
`513`	`515`	`do_sve_acc(esr, regs);`
	`516`	`+ exit_to_user_mode(regs);`
`514`	`517`	`}`
`515`	`518`
`516`	`519`	`static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)`
`517`	`520`	`{`
`518`		`- enter_from_user_mode();`
	`521`	`+ enter_from_user_mode(regs);`
`519`	`522`	`local_daif_restore(DAIF_PROCCTX);`
`520`	`523`	`do_fpsimd_exc(esr, regs);`
	`524`	`+ exit_to_user_mode(regs);`
`521`	`525`	`}`
`522`	`526`
`523`	`527`	`static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)`
`524`	`528`	`{`
`525`		`- enter_from_user_mode();`
	`529`	`+ enter_from_user_mode(regs);`
`526`	`530`	`local_daif_restore(DAIF_PROCCTX);`
`527`	`531`	`do_sysinstr(esr, regs);`
	`532`	`+ exit_to_user_mode(regs);`
`528`	`533`	`}`
`529`	`534`
`530`	`535`	`static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)`
`@@ -534,61 +539,69 @@ static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)`
`534`	`539`	`if (!is_ttbr0_addr(instruction_pointer(regs)))`
`535`	`540`	`arm64_apply_bp_hardening();`
`536`	`541`
`537`		`- enter_from_user_mode();`
	`542`	`+ enter_from_user_mode(regs);`
`538`	`543`	`local_daif_restore(DAIF_PROCCTX);`
`539`	`544`	`do_sp_pc_abort(far, esr, regs);`
	`545`	`+ exit_to_user_mode(regs);`
`540`	`546`	`}`
`541`	`547`
`542`	`548`	`static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)`
`543`	`549`	`{`
`544`		`- enter_from_user_mode();`
	`550`	`+ enter_from_user_mode(regs);`
`545`	`551`	`local_daif_restore(DAIF_PROCCTX);`
`546`	`552`	`do_sp_pc_abort(regs->sp, esr, regs);`
	`553`	`+ exit_to_user_mode(regs);`
`547`	`554`	`}`
`548`	`555`
`549`	`556`	`static void noinstr el0_undef(struct pt_regs *regs)`
`550`	`557`	`{`
`551`		`- enter_from_user_mode();`
	`558`	`+ enter_from_user_mode(regs);`
`552`	`559`	`local_daif_restore(DAIF_PROCCTX);`
`553`	`560`	`do_undefinstr(regs);`
	`561`	`+ exit_to_user_mode(regs);`
`554`	`562`	`}`
`555`	`563`
`556`	`564`	`static void noinstr el0_bti(struct pt_regs *regs)`
`557`	`565`	`{`
`558`		`- enter_from_user_mode();`
	`566`	`+ enter_from_user_mode(regs);`
`559`	`567`	`local_daif_restore(DAIF_PROCCTX);`
`560`	`568`	`do_bti(regs);`
	`569`	`+ exit_to_user_mode(regs);`
`561`	`570`	`}`
`562`	`571`
`563`	`572`	`static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)`
`564`	`573`	`{`
`565`		`- enter_from_user_mode();`
	`574`	`+ enter_from_user_mode(regs);`
`566`	`575`	`local_daif_restore(DAIF_PROCCTX);`
`567`	`576`	`bad_el0_sync(regs, 0, esr);`
	`577`	`+ exit_to_user_mode(regs);`
`568`	`578`	`}`
`569`	`579`
`570`	`580`	`static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)`
`571`	`581`	`{`
`572`	`582`	`/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */`
`573`	`583`	`unsigned long far = read_sysreg(far_el1);`
`574`	`584`
`575`		`- enter_from_user_mode();`
	`585`	`+ enter_from_user_mode(regs);`
`576`	`586`	`do_debug_exception(far, esr, regs);`
`577`	`587`	`local_daif_restore(DAIF_PROCCTX);`
	`588`	`+ exit_to_user_mode(regs);`
`578`	`589`	`}`
`579`	`590`
`580`	`591`	`static void noinstr el0_svc(struct pt_regs *regs)`
`581`	`592`	`{`
`582`		`- enter_from_user_mode();`
	`593`	`+ enter_from_user_mode(regs);`
`583`	`594`	`cortex_a76_erratum_1463225_svc_handler();`
`584`	`595`	`do_el0_svc(regs);`
	`596`	`+ exit_to_user_mode(regs);`
`585`	`597`	`}`
`586`	`598`
`587`	`599`	`static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)`
`588`	`600`	`{`
`589`		`- enter_from_user_mode();`
	`601`	`+ enter_from_user_mode(regs);`
`590`	`602`	`local_daif_restore(DAIF_PROCCTX);`
`591`	`603`	`do_ptrauth_fault(regs, esr);`
	`604`	`+ exit_to_user_mode(regs);`
`592`	`605`	`}`
`593`	`606`
`594`	`607`	`asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)`
`@@ -647,14 +660,16 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)`
`647`	`660`	`static void noinstr el0_interrupt(struct pt_regs *regs,`
`648`	`661`	`void (handler)(struct pt_regs ))`
`649`	`662`	`{`
`650`		`- enter_from_user_mode();`
	`663`	`+ enter_from_user_mode(regs);`
`651`	`664`
`652`	`665`	`write_sysreg(DAIF_PROCCTX_NOIRQ, daif);`
`653`	`666`
`654`	`667`	`if (regs->pc & BIT(55))`
`655`	`668`	`arm64_apply_bp_hardening();`
`656`	`669`
`657`	`670`	`do_interrupt_handler(regs, handler);`
	`671`	`+`
	`672`	`+ exit_to_user_mode(regs);`
`658`	`673`	`}`
`659`	`674`
`660`	`675`	`static void noinstr __el0_irq_handler_common(struct pt_regs *regs)`
`@@ -681,12 +696,13 @@ static void noinstr __el0_error_handler_common(struct pt_regs *regs)`
`681`	`696`	`{`
`682`	`697`	`unsigned long esr = read_sysreg(esr_el1);`
`683`	`698`
`684`		`- enter_from_user_mode();`
	`699`	`+ enter_from_user_mode(regs);`
`685`	`700`	`local_daif_restore(DAIF_ERRCTX);`
`686`	`701`	`arm64_enter_nmi(regs);`
`687`	`702`	`do_serror(regs, esr);`
`688`	`703`	`arm64_exit_nmi(regs);`
`689`	`704`	`local_daif_restore(DAIF_PROCCTX);`
	`705`	`+ exit_to_user_mode(regs);`
`690`	`706`	`}`
`691`	`707`
`692`	`708`	`asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)`
`@@ -697,16 +713,18 @@ asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)`
`697`	`713`	`#ifdef CONFIG_COMPAT`
`698`	`714`	`static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)`
`699`	`715`	`{`
`700`		`- enter_from_user_mode();`
	`716`	`+ enter_from_user_mode(regs);`
`701`	`717`	`local_daif_restore(DAIF_PROCCTX);`
`702`	`718`	`do_cp15instr(esr, regs);`
	`719`	`+ exit_to_user_mode(regs);`
`703`	`720`	`}`
`704`	`721`
`705`	`722`	`static void noinstr el0_svc_compat(struct pt_regs *regs)`
`706`	`723`	`{`
`707`		`- enter_from_user_mode();`
	`724`	`+ enter_from_user_mode(regs);`
`708`	`725`	`cortex_a76_erratum_1463225_svc_handler();`
`709`	`726`	`do_el0_svc_compat(regs);`
	`727`	`+ exit_to_user_mode(regs);`
`710`	`728`	`}`
`711`	`729`
`712`	`730`	`asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)`