Skip to content

Commit 1270dad

Browse files
ouptonoupton
committed
Merge branch kvm-arm64/el2-kcfi into kvmarm/next
* kvm-arm64/el2-kcfi: : kCFI support in the EL2 hypervisor, courtesy of Pierre-Clément Tosi : : Enable the usage fo CONFIG_CFI_CLANG (kCFI) for hardening indirect : branches in the EL2 hypervisor. Unlike kernel support for the feature, : CFI failures at EL2 are always fatal. KVM: arm64: nVHE: Support CONFIG_CFI_CLANG at EL2 KVM: arm64: Introduce print_nvhe_hyp_panic helper arm64: Introduce esr_brk_comment, esr_is_cfi_brk KVM: arm64: VHE: Mark __hyp_call_panic __noreturn KVM: arm64: nVHE: gen-hyprel: Skip R_AARCH64_ABS32 KVM: arm64: nVHE: Simplify invalid_host_el2_vect KVM: arm64: Fix __pkvm_init_switch_pgd call ABI KVM: arm64: Fix clobbered ELR in sync abort/SError Signed-off-by: Oliver Upton <[email protected]>
2 parents 377d0e5 + eca4ba5 commit 1270dad

File tree

14 files changed

+78
-42
lines changed

14 files changed

+78
-42
lines changed

‎arch/arm64/include/asm/esr.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -380,13 +380,24 @@
380380
#ifndef __ASSEMBLY__
381381
#include <asm/types.h>
382382

383+
static inline unsigned long esr_brk_comment(unsigned long esr)
384+
{
385+
return esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
386+
}
387+
383388
static inline bool esr_is_data_abort(unsigned long esr)
384389
{
385390
const unsigned long ec = ESR_ELx_EC(esr);
386391

387392
return ec == ESR_ELx_EC_DABT_LOW || ec == ESR_ELx_EC_DABT_CUR;
388393
}
389394

395+
static inline bool esr_is_cfi_brk(unsigned long esr)
396+
{
397+
return ESR_ELx_EC(esr) == ESR_ELx_EC_BRK64 &&
398+
(esr_brk_comment(esr) & ~CFI_BRK_IMM_MASK) == CFI_BRK_IMM_BASE;
399+
}
400+
390401
static inline bool esr_fsc_is_translation_fault(unsigned long esr)
391402
{
392403
/* Translation fault, level -1 */

‎arch/arm64/include/asm/kvm_hyp.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,8 @@ void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr,
124124
#endif
125125

126126
#ifdef __KVM_NVHE_HYPERVISOR__
127-
void __pkvm_init_switch_pgd(phys_addr_t phys, unsigned long size,
128-
phys_addr_t pgd, void *sp, void *cont_fn);
127+
void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp,
128+
void (*fn)(void));
129129
int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
130130
unsigned long *per_cpu_base, u32 hyp_va_bits);
131131
void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);

‎arch/arm64/kernel/asm-offsets.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ int main(void)
128128
DEFINE(VCPU_FAULT_DISR, offsetof(struct kvm_vcpu, arch.fault.disr_el1));
129129
DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2));
130130
DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_cpu_context, regs));
131+
DEFINE(CPU_ELR_EL2, offsetof(struct kvm_cpu_context, sys_regs[ELR_EL2]));
131132
DEFINE(CPU_RGSR_EL1, offsetof(struct kvm_cpu_context, sys_regs[RGSR_EL1]));
132133
DEFINE(CPU_GCR_EL1, offsetof(struct kvm_cpu_context, sys_regs[GCR_EL1]));
133134
DEFINE(CPU_APIAKEYLO_EL1, offsetof(struct kvm_cpu_context, sys_regs[APIAKEYLO_EL1]));

‎arch/arm64/kernel/debug-monitors.c

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -312,9 +312,7 @@ static int call_break_hook(struct pt_regs *regs, unsigned long esr)
312312
* entirely not preemptible, and we can use rcu list safely here.
313313
*/
314314
list_for_each_entry_rcu(hook, list, node) {
315-
unsigned long comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
316-
317-
if ((comment & ~hook->mask) == hook->imm)
315+
if ((esr_brk_comment(esr) & ~hook->mask) == hook->imm)
318316
fn = hook->fn;
319317
}
320318

‎arch/arm64/kernel/traps.c

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1105,8 +1105,6 @@ static struct break_hook ubsan_break_hook = {
11051105
};
11061106
#endif
11071107

1108-
#define esr_comment(esr) ((esr) & ESR_ELx_BRK64_ISS_COMMENT_MASK)
1109-
11101108
/*
11111109
* Initial handler for AArch64 BRK exceptions
11121110
* This handler only used until debug_traps_init().
@@ -1115,15 +1113,15 @@ int __init early_brk64(unsigned long addr, unsigned long esr,
11151113
struct pt_regs *regs)
11161114
{
11171115
#ifdef CONFIG_CFI_CLANG
1118-
if ((esr_comment(esr) & ~CFI_BRK_IMM_MASK) == CFI_BRK_IMM_BASE)
1116+
if (esr_is_cfi_brk(esr))
11191117
return cfi_handler(regs, esr) != DBG_HOOK_HANDLED;
11201118
#endif
11211119
#ifdef CONFIG_KASAN_SW_TAGS
1122-
if ((esr_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
1120+
if ((esr_brk_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
11231121
return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
11241122
#endif
11251123
#ifdef CONFIG_UBSAN_TRAP
1126-
if ((esr_comment(esr) & ~UBSAN_BRK_MASK) == UBSAN_BRK_IMM)
1124+
if ((esr_brk_comment(esr) & ~UBSAN_BRK_MASK) == UBSAN_BRK_IMM)
11271125
return ubsan_handler(regs, esr) != DBG_HOOK_HANDLED;
11281126
#endif
11291127
return bug_handler(regs, esr) != DBG_HOOK_HANDLED;

‎arch/arm64/kvm/handle_exit.c

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,20 @@ void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index)
411411
kvm_handle_guest_serror(vcpu, kvm_vcpu_get_esr(vcpu));
412412
}
413413

414+
static void print_nvhe_hyp_panic(const char *name, u64 panic_addr)
415+
{
416+
kvm_err("nVHE hyp %s at: [<%016llx>] %pB!\n", name, panic_addr,
417+
(void *)(panic_addr + kaslr_offset()));
418+
}
419+
420+
static void kvm_nvhe_report_cfi_failure(u64 panic_addr)
421+
{
422+
print_nvhe_hyp_panic("CFI failure", panic_addr);
423+
424+
if (IS_ENABLED(CONFIG_CFI_PERMISSIVE))
425+
kvm_err(" (CONFIG_CFI_PERMISSIVE ignored for hyp failures)\n");
426+
}
427+
414428
void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr,
415429
u64 elr_virt, u64 elr_phys,
416430
u64 par, uintptr_t vcpu,
@@ -423,7 +437,7 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr,
423437
if (mode != PSR_MODE_EL2t && mode != PSR_MODE_EL2h) {
424438
kvm_err("Invalid host exception to nVHE hyp!\n");
425439
} else if (ESR_ELx_EC(esr) == ESR_ELx_EC_BRK64 &&
426-
(esr & ESR_ELx_BRK64_ISS_COMMENT_MASK) == BUG_BRK_IMM) {
440+
esr_brk_comment(esr) == BUG_BRK_IMM) {
427441
const char *file = NULL;
428442
unsigned int line = 0;
429443

@@ -439,11 +453,11 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr,
439453
if (file)
440454
kvm_err("nVHE hyp BUG at: %s:%u!\n", file, line);
441455
else
442-
kvm_err("nVHE hyp BUG at: [<%016llx>] %pB!\n", panic_addr,
443-
(void *)(panic_addr + kaslr_offset()));
456+
print_nvhe_hyp_panic("BUG", panic_addr);
457+
} else if (IS_ENABLED(CONFIG_CFI_CLANG) && esr_is_cfi_brk(esr)) {
458+
kvm_nvhe_report_cfi_failure(panic_addr);
444459
} else {
445-
kvm_err("nVHE hyp panic at: [<%016llx>] %pB!\n", panic_addr,
446-
(void *)(panic_addr + kaslr_offset()));
460+
print_nvhe_hyp_panic("panic", panic_addr);
447461
}
448462

449463
/* Dump the nVHE hypervisor backtrace */

‎arch/arm64/kvm/hyp/entry.S

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,14 @@ alternative_else_nop_endif
8383
eret
8484
sb
8585

86+
SYM_INNER_LABEL(__guest_exit_restore_elr_and_panic, SYM_L_GLOBAL)
87+
// x2-x29,lr: vcpu regs
88+
// vcpu x0-x1 on the stack
89+
90+
adr_this_cpu x0, kvm_hyp_ctxt, x1
91+
ldr x0, [x0, #CPU_ELR_EL2]
92+
msr elr_el2, x0
93+
8694
SYM_INNER_LABEL(__guest_exit_panic, SYM_L_GLOBAL)
8795
// x2-x29,lr: vcpu regs
8896
// vcpu x0-x1 on the stack

‎arch/arm64/kvm/hyp/include/hyp/switch.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -693,7 +693,7 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
693693

694694
static inline void __kvm_unexpected_el2_exception(void)
695695
{
696-
extern char __guest_exit_panic[];
696+
extern char __guest_exit_restore_elr_and_panic[];
697697
unsigned long addr, fixup;
698698
struct kvm_exception_table_entry *entry, *end;
699699
unsigned long elr_el2 = read_sysreg(elr_el2);
@@ -715,7 +715,8 @@ static inline void __kvm_unexpected_el2_exception(void)
715715
}
716716

717717
/* Trigger a panic after restoring the hyp context. */
718-
write_sysreg(__guest_exit_panic, elr_el2);
718+
this_cpu_ptr(&kvm_hyp_ctxt)->sys_regs[ELR_EL2] = elr_el2;
719+
write_sysreg(__guest_exit_restore_elr_and_panic, elr_el2);
719720
}
720721

721722
#endif /* __ARM64_KVM_HYP_SWITCH_H__ */

‎arch/arm64/kvm/hyp/nvhe/Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,9 @@ quiet_cmd_hyprel = HYPREL $@
8989
quiet_cmd_hypcopy = HYPCOPY $@
9090
cmd_hypcopy = $(OBJCOPY) --prefix-symbols=__kvm_nvhe_ $< $@
9191

92-
# Remove ftrace, Shadow Call Stack, and CFI CFLAGS.
93-
# This is equivalent to the 'notrace', '__noscs', and '__nocfi' annotations.
94-
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_CFI), $(KBUILD_CFLAGS))
92+
# Remove ftrace and Shadow Call Stack CFLAGS.
93+
# This is equivalent to the 'notrace' and '__noscs' annotations.
94+
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
9595
# Starting from 13.0.0 llvm emits SHT_REL section '.llvm.call-graph-profile'
9696
# when profile optimization is applied. gen-hyprel does not support SHT_REL and
9797
# causes a build failure. Remove profile optimization flags.

‎arch/arm64/kvm/hyp/nvhe/gen-hyprel.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@
5050
#ifndef R_AARCH64_ABS64
5151
#define R_AARCH64_ABS64 257
5252
#endif
53+
#ifndef R_AARCH64_ABS32
54+
#define R_AARCH64_ABS32 258
55+
#endif
5356
#ifndef R_AARCH64_PREL64
5457
#define R_AARCH64_PREL64 260
5558
#endif
@@ -383,6 +386,9 @@ static void emit_rela_section(Elf64_Shdr *sh_rela)
383386
case R_AARCH64_ABS64:
384387
emit_rela_abs64(rela, sh_orig_name);
385388
break;
389+
/* Allow 32-bit absolute relocation, for kCFI type hashes. */
390+
case R_AARCH64_ABS32:
391+
break;
386392
/* Allow position-relative data relocations. */
387393
case R_AARCH64_PREL64:
388394
case R_AARCH64_PREL32:

0 commit comments

Comments
 (0)