Skip to content

Commit 13aeb9b

Browse files
David BrazdilMarc Zyngier
David Brazdil
authored and
Marc Zyngier
committed
KVM: arm64: Split hyp/sysreg-sr.c to VHE/nVHE
sysreg-sr.c contains KVM's code for saving/restoring system registers, with some code shared between VHE/nVHE. These common routines are moved to a header file, VHE-specific code is moved to vhe/sysreg-sr.c and nVHE-specific code to nvhe/sysreg-sr.c. Signed-off-by: David Brazdil <[email protected]> Signed-off-by: Marc Zyngier <[email protected]> Link: https://lore.kernel.org/r/[email protected]
1 parent d400c5b commit 13aeb9b

File tree

12 files changed

+191
-164
lines changed

12 files changed

+191
-164
lines changed

‎arch/arm64/include/asm/kvm_host.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ struct kvm_vcpu_arch {
338338
struct vcpu_reset_state reset_state;
339339

340340
/* True when deferrable sysregs are loaded on the physical CPU,
341-
* see kvm_vcpu_load_sysregs and kvm_vcpu_put_sysregs. */
341+
* see kvm_vcpu_load_sysregs_vhe and kvm_vcpu_put_sysregs_vhe. */
342342
bool sysregs_loaded_on_cpu;
343343

344344
/* Guest PV state */
@@ -639,8 +639,8 @@ static inline int kvm_arm_have_ssbd(void)
639639
}
640640
}
641641

642-
void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu);
643-
void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu);
642+
void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu);
643+
void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu);
644644

645645
int kvm_set_ipa_limit(void);
646646

‎arch/arm64/include/asm/kvm_hyp.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,15 @@ int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu);
6666
void __timer_enable_traps(struct kvm_vcpu *vcpu);
6767
void __timer_disable_traps(struct kvm_vcpu *vcpu);
6868

69+
#ifdef __KVM_NVHE_HYPERVISOR__
6970
void __sysreg_save_state_nvhe(struct kvm_cpu_context *ctxt);
7071
void __sysreg_restore_state_nvhe(struct kvm_cpu_context *ctxt);
72+
#else
7173
void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt);
7274
void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt);
7375
void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt);
7476
void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt);
75-
void __sysreg32_save_state(struct kvm_vcpu *vcpu);
76-
void __sysreg32_restore_state(struct kvm_vcpu *vcpu);
77+
#endif
7778

7879
void __debug_switch_to_guest(struct kvm_vcpu *vcpu);
7980
void __debug_switch_to_host(struct kvm_vcpu *vcpu);

‎arch/arm64/kernel/image-vars.h

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,6 @@ KVM_NVHE_ALIAS(abort_guest_exit_start);
7676
KVM_NVHE_ALIAS(__fpsimd_restore_state);
7777
KVM_NVHE_ALIAS(__fpsimd_save_state);
7878

79-
/* Symbols defined in sysreg-sr.c (not yet compiled with nVHE build rules). */
80-
KVM_NVHE_ALIAS(__kvm_enable_ssbs);
81-
KVM_NVHE_ALIAS(__sysreg32_restore_state);
82-
KVM_NVHE_ALIAS(__sysreg32_save_state);
83-
KVM_NVHE_ALIAS(__sysreg_restore_state_nvhe);
84-
KVM_NVHE_ALIAS(__sysreg_save_state_nvhe);
85-
8679
/* Symbols defined in timer-sr.c (not yet compiled with nVHE build rules). */
8780
KVM_NVHE_ALIAS(__kvm_timer_set_cntvoff);
8881
KVM_NVHE_ALIAS(__timer_disable_traps);

‎arch/arm64/kvm/arm.c

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
351351

352352
kvm_vgic_load(vcpu);
353353
kvm_timer_vcpu_load(vcpu);
354-
kvm_vcpu_load_sysregs(vcpu);
354+
if (has_vhe())
355+
kvm_vcpu_load_sysregs_vhe(vcpu);
355356
kvm_arch_vcpu_load_fp(vcpu);
356357
kvm_vcpu_pmu_restore_guest(vcpu);
357358
if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
@@ -369,7 +370,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
369370
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
370371
{
371372
kvm_arch_vcpu_put_fp(vcpu);
372-
kvm_vcpu_put_sysregs(vcpu);
373+
if (has_vhe())
374+
kvm_vcpu_put_sysregs_vhe(vcpu);
373375
kvm_timer_vcpu_put(vcpu);
374376
kvm_vgic_put(vcpu);
375377
kvm_vcpu_pmu_restore_host(vcpu);
@@ -1302,7 +1304,7 @@ static void cpu_init_hyp_mode(void)
13021304
*/
13031305
if (this_cpu_has_cap(ARM64_SSBS) &&
13041306
arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {
1305-
kvm_call_hyp(__kvm_enable_ssbs);
1307+
kvm_call_hyp_nvhe(__kvm_enable_ssbs);
13061308
}
13071309
}
13081310

‎arch/arm64/kvm/hyp/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ subdir-ccflags-y := -I$(incdir) \
1313
obj-$(CONFIG_KVM) += hyp.o vhe/ nvhe/
1414
obj-$(CONFIG_KVM_INDIRECT_VECTORS) += smccc_wa.o
1515

16-
hyp-y := vgic-v3-sr.o timer-sr.o aarch32.o vgic-v2-cpuif-proxy.o sysreg-sr.o \
17-
entry.o fpsimd.o
16+
hyp-y := vgic-v3-sr.o timer-sr.o aarch32.o vgic-v2-cpuif-proxy.o entry.o \
17+
fpsimd.o
1818

1919
# KVM code is run at a different exception code with a different map, so
2020
# compiler instrumentation that inserts callbacks or checks into the code may

‎arch/arm64/kvm/hyp/sysreg-sr.c renamed to ‎arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h

Lines changed: 14 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
* Author: Marc Zyngier <[email protected]>
55
*/
66

7+
#ifndef __ARM64_KVM_HYP_SYSREG_SR_H__
8+
#define __ARM64_KVM_HYP_SYSREG_SR_H__
9+
710
#include <linux/compiler.h>
811
#include <linux/kvm_host.h>
912

@@ -12,30 +15,18 @@
1215
#include <asm/kvm_emulate.h>
1316
#include <asm/kvm_hyp.h>
1417

15-
/*
16-
* Non-VHE: Both host and guest must save everything.
17-
*
18-
* VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
19-
* pstate, which are handled as part of the el2 return state) on every
20-
* switch (sp_el0 is being dealt with in the assembly code).
21-
* tpidr_el0 and tpidrro_el0 only need to be switched when going
22-
* to host userspace or a different VCPU. EL1 registers only need to be
23-
* switched when potentially going to run a different VCPU. The latter two
24-
* classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put.
25-
*/
26-
27-
static void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
18+
static inline void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
2819
{
2920
ctxt->sys_regs[MDSCR_EL1] = read_sysreg(mdscr_el1);
3021
}
3122

32-
static void __hyp_text __sysreg_save_user_state(struct kvm_cpu_context *ctxt)
23+
static inline void __hyp_text __sysreg_save_user_state(struct kvm_cpu_context *ctxt)
3324
{
3425
ctxt->sys_regs[TPIDR_EL0] = read_sysreg(tpidr_el0);
3526
ctxt->sys_regs[TPIDRRO_EL0] = read_sysreg(tpidrro_el0);
3627
}
3728

38-
static void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
29+
static inline void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
3930
{
4031
ctxt->sys_regs[CSSELR_EL1] = read_sysreg(csselr_el1);
4132
ctxt->sys_regs[SCTLR_EL1] = read_sysreg_el1(SYS_SCTLR);
@@ -60,7 +51,7 @@ static void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
6051
ctxt->gp_regs.spsr[KVM_SPSR_EL1]= read_sysreg_el1(SYS_SPSR);
6152
}
6253

63-
static void __hyp_text __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
54+
static inline void __hyp_text __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
6455
{
6556
ctxt->gp_regs.regs.pc = read_sysreg_el2(SYS_ELR);
6657
ctxt->gp_regs.regs.pstate = read_sysreg_el2(SYS_SPSR);
@@ -69,39 +60,18 @@ static void __hyp_text __sysreg_save_el2_return_state(struct kvm_cpu_context *ct
6960
ctxt->sys_regs[DISR_EL1] = read_sysreg_s(SYS_VDISR_EL2);
7061
}
7162

72-
void __hyp_text __sysreg_save_state_nvhe(struct kvm_cpu_context *ctxt)
73-
{
74-
__sysreg_save_el1_state(ctxt);
75-
__sysreg_save_common_state(ctxt);
76-
__sysreg_save_user_state(ctxt);
77-
__sysreg_save_el2_return_state(ctxt);
78-
}
79-
80-
void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
81-
{
82-
__sysreg_save_common_state(ctxt);
83-
}
84-
NOKPROBE_SYMBOL(sysreg_save_host_state_vhe);
85-
86-
void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
87-
{
88-
__sysreg_save_common_state(ctxt);
89-
__sysreg_save_el2_return_state(ctxt);
90-
}
91-
NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe);
92-
93-
static void __hyp_text __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
63+
static inline void __hyp_text __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
9464
{
9565
write_sysreg(ctxt->sys_regs[MDSCR_EL1], mdscr_el1);
9666
}
9767

98-
static void __hyp_text __sysreg_restore_user_state(struct kvm_cpu_context *ctxt)
68+
static inline void __hyp_text __sysreg_restore_user_state(struct kvm_cpu_context *ctxt)
9969
{
10070
write_sysreg(ctxt->sys_regs[TPIDR_EL0], tpidr_el0);
10171
write_sysreg(ctxt->sys_regs[TPIDRRO_EL0], tpidrro_el0);
10272
}
10373

104-
static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
74+
static inline void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
10575
{
10676
write_sysreg(ctxt->sys_regs[MPIDR_EL1], vmpidr_el2);
10777
write_sysreg(ctxt->sys_regs[CSSELR_EL1], csselr_el1);
@@ -160,8 +130,7 @@ static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
160130
write_sysreg_el1(ctxt->gp_regs.spsr[KVM_SPSR_EL1],SYS_SPSR);
161131
}
162132

163-
static void __hyp_text
164-
__sysreg_restore_el2_return_state(struct kvm_cpu_context *ctxt)
133+
static inline void __hyp_text __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctxt)
165134
{
166135
u64 pstate = ctxt->gp_regs.regs.pstate;
167136
u64 mode = pstate & PSR_AA32_MODE_MASK;
@@ -187,28 +156,7 @@ __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctxt)
187156
write_sysreg_s(ctxt->sys_regs[DISR_EL1], SYS_VDISR_EL2);
188157
}
189158

190-
void __hyp_text __sysreg_restore_state_nvhe(struct kvm_cpu_context *ctxt)
191-
{
192-
__sysreg_restore_el1_state(ctxt);
193-
__sysreg_restore_common_state(ctxt);
194-
__sysreg_restore_user_state(ctxt);
195-
__sysreg_restore_el2_return_state(ctxt);
196-
}
197-
198-
void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
199-
{
200-
__sysreg_restore_common_state(ctxt);
201-
}
202-
NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe);
203-
204-
void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
205-
{
206-
__sysreg_restore_common_state(ctxt);
207-
__sysreg_restore_el2_return_state(ctxt);
208-
}
209-
NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe);
210-
211-
void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
159+
static inline void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
212160
{
213161
u64 *spsr, *sysreg;
214162

@@ -230,7 +178,7 @@ void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
230178
sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
231179
}
232180

233-
void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
181+
static inline void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
234182
{
235183
u64 *spsr, *sysreg;
236184

@@ -252,82 +200,4 @@ void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
252200
write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
253201
}
254202

255-
/**
256-
* kvm_vcpu_load_sysregs - Load guest system registers to the physical CPU
257-
*
258-
* @vcpu: The VCPU pointer
259-
*
260-
* Load system registers that do not affect the host's execution, for
261-
* example EL1 system registers on a VHE system where the host kernel
262-
* runs at EL2. This function is called from KVM's vcpu_load() function
263-
* and loading system register state early avoids having to load them on
264-
* every entry to the VM.
265-
*/
266-
void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu)
267-
{
268-
struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
269-
struct kvm_cpu_context *host_ctxt;
270-
271-
if (!has_vhe())
272-
return;
273-
274-
host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
275-
__sysreg_save_user_state(host_ctxt);
276-
277-
/*
278-
* Load guest EL1 and user state
279-
*
280-
* We must restore the 32-bit state before the sysregs, thanks
281-
* to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
282-
*/
283-
__sysreg32_restore_state(vcpu);
284-
__sysreg_restore_user_state(guest_ctxt);
285-
__sysreg_restore_el1_state(guest_ctxt);
286-
287-
vcpu->arch.sysregs_loaded_on_cpu = true;
288-
289-
activate_traps_vhe_load(vcpu);
290-
}
291-
292-
/**
293-
* kvm_vcpu_put_sysregs - Restore host system registers to the physical CPU
294-
*
295-
* @vcpu: The VCPU pointer
296-
*
297-
* Save guest system registers that do not affect the host's execution, for
298-
* example EL1 system registers on a VHE system where the host kernel
299-
* runs at EL2. This function is called from KVM's vcpu_put() function
300-
* and deferring saving system register state until we're no longer running the
301-
* VCPU avoids having to save them on every exit from the VM.
302-
*/
303-
void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu)
304-
{
305-
struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
306-
struct kvm_cpu_context *host_ctxt;
307-
308-
if (!has_vhe())
309-
return;
310-
311-
host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
312-
deactivate_traps_vhe_put();
313-
314-
__sysreg_save_el1_state(guest_ctxt);
315-
__sysreg_save_user_state(guest_ctxt);
316-
__sysreg32_save_state(vcpu);
317-
318-
/* Restore host user state */
319-
__sysreg_restore_user_state(host_ctxt);
320-
321-
vcpu->arch.sysregs_loaded_on_cpu = false;
322-
}
323-
324-
void __hyp_text __kvm_enable_ssbs(void)
325-
{
326-
u64 tmp;
327-
328-
asm volatile(
329-
"mrs %0, sctlr_el2\n"
330-
"orr %0, %0, %1\n"
331-
"msr sctlr_el2, %0"
332-
: "=&r" (tmp) : "L" (SCTLR_ELx_DSSBS));
333-
}
203+
#endif /* __ARM64_KVM_HYP_SYSREG_SR_H__ */

‎arch/arm64/kvm/hyp/nvhe/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
asflags-y := -D__KVM_NVHE_HYPERVISOR__
77
ccflags-y := -D__KVM_NVHE_HYPERVISOR__
88

9-
obj-y := debug-sr.o switch.o tlb.o hyp-init.o ../hyp-entry.o
9+
obj-y := sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o ../hyp-entry.o
1010

1111
obj-y := $(patsubst %.o,%.hyp.o,$(obj-y))
1212
extra-y := $(patsubst %.hyp.o,%.hyp.tmp.o,$(obj-y))

‎arch/arm64/kvm/hyp/nvhe/switch.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*/
66

77
#include <hyp/switch.h>
8+
#include <hyp/sysreg-sr.h>
89

910
#include <linux/arm-smccc.h>
1011
#include <linux/kvm_host.h>

‎arch/arm64/kvm/hyp/nvhe/sysreg-sr.c

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/*
3+
* Copyright (C) 2012-2015 - ARM Ltd
4+
* Author: Marc Zyngier <[email protected]>
5+
*/
6+
7+
#include <hyp/sysreg-sr.h>
8+
9+
#include <linux/compiler.h>
10+
#include <linux/kvm_host.h>
11+
12+
#include <asm/kprobes.h>
13+
#include <asm/kvm_asm.h>
14+
#include <asm/kvm_emulate.h>
15+
#include <asm/kvm_hyp.h>
16+
17+
/*
18+
* Non-VHE: Both host and guest must save everything.
19+
*/
20+
21+
void __hyp_text __sysreg_save_state_nvhe(struct kvm_cpu_context *ctxt)
22+
{
23+
__sysreg_save_el1_state(ctxt);
24+
__sysreg_save_common_state(ctxt);
25+
__sysreg_save_user_state(ctxt);
26+
__sysreg_save_el2_return_state(ctxt);
27+
}
28+
29+
void __hyp_text __sysreg_restore_state_nvhe(struct kvm_cpu_context *ctxt)
30+
{
31+
__sysreg_restore_el1_state(ctxt);
32+
__sysreg_restore_common_state(ctxt);
33+
__sysreg_restore_user_state(ctxt);
34+
__sysreg_restore_el2_return_state(ctxt);
35+
}
36+
37+
void __hyp_text __kvm_enable_ssbs(void)
38+
{
39+
u64 tmp;
40+
41+
asm volatile(
42+
"mrs %0, sctlr_el2\n"
43+
"orr %0, %0, %1\n"
44+
"msr sctlr_el2, %0"
45+
: "=&r" (tmp) : "L" (SCTLR_ELx_DSSBS));
46+
}

‎arch/arm64/kvm/hyp/vhe/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
asflags-y := -D__KVM_VHE_HYPERVISOR__
77
ccflags-y := -D__KVM_VHE_HYPERVISOR__
88

9-
obj-y := debug-sr.o switch.o tlb.o ../hyp-entry.o
9+
obj-y := sysreg-sr.o debug-sr.o switch.o tlb.o ../hyp-entry.o
1010

1111
# KVM code is run at a different exception code with a different map, so
1212
# compiler instrumentation that inserts callbacks or checks into the code may

0 commit comments

Comments
 (0)