Skip to content

Commit 7db99f0

Browse files
torvaldstorvalds
committed
Merge tag 'x86_cpu_for_v6.1_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 cpu updates from Borislav Petkov: - Print the CPU number at segfault time. The number printed is not always accurate (preemption is enabled at that time) but the print string contains "likely" and after a lot of back'n'forth on this, this was the consensus that was reached. See thread at [1]. - After a *lot* of testing and polishing, finally the clear_user() improvements to inline REP; STOSB by default Link: https://lore.kernel.org/r/[email protected] [1] * tag 'x86_cpu_for_v6.1_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/mm: Print likely CPU at segfault time x86/clear_user: Make it faster
2 parents ba94a7a + c926087 commit 7db99f0

File tree

6 files changed

+198
-43
lines changed

6 files changed

+198
-43
lines changed

‎arch/x86/include/asm/uaccess.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -502,9 +502,6 @@ strncpy_from_user(char *dst, const char __user *src, long count);
502502

503503
extern __must_check long strnlen_user(const char __user *str, long n);
504504

505-
unsigned long __must_check clear_user(void __user *mem, unsigned long len);
506-
unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
507-
508505
#ifdef CONFIG_ARCH_HAS_COPY_MC
509506
unsigned long __must_check
510507
copy_mc_to_kernel(void *to, const void *from, unsigned len);
@@ -526,6 +523,8 @@ extern struct movsl_mask {
526523
#define ARCH_HAS_NOCACHE_UACCESS 1
527524

528525
#ifdef CONFIG_X86_32
526+
unsigned long __must_check clear_user(void __user *mem, unsigned long len);
527+
unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
529528
# include <asm/uaccess_32.h>
530529
#else
531530
# include <asm/uaccess_64.h>

‎arch/x86/include/asm/uaccess_64.h

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,4 +79,49 @@ __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
7979
kasan_check_write(dst, size);
8080
return __copy_user_flushcache(dst, src, size);
8181
}
82+
83+
/*
84+
* Zero Userspace.
85+
*/
86+
87+
__must_check unsigned long
88+
clear_user_original(void __user *addr, unsigned long len);
89+
__must_check unsigned long
90+
clear_user_rep_good(void __user *addr, unsigned long len);
91+
__must_check unsigned long
92+
clear_user_erms(void __user *addr, unsigned long len);
93+
94+
static __always_inline __must_check unsigned long __clear_user(void __user *addr, unsigned long size)
95+
{
96+
might_fault();
97+
stac();
98+
99+
/*
100+
* No memory constraint because it doesn't change any memory gcc
101+
* knows about.
102+
*/
103+
asm volatile(
104+
"1:\n\t"
105+
ALTERNATIVE_3("rep stosb",
106+
"call clear_user_erms", ALT_NOT(X86_FEATURE_FSRM),
107+
"call clear_user_rep_good", ALT_NOT(X86_FEATURE_ERMS),
108+
"call clear_user_original", ALT_NOT(X86_FEATURE_REP_GOOD))
109+
"2:\n"
110+
_ASM_EXTABLE_UA(1b, 2b)
111+
: "+c" (size), "+D" (addr), ASM_CALL_CONSTRAINT
112+
: "a" (0)
113+
/* rep_good clobbers %rdx */
114+
: "rdx");
115+
116+
clac();
117+
118+
return size;
119+
}
120+
121+
static __always_inline unsigned long clear_user(void __user *to, unsigned long n)
122+
{
123+
if (access_ok(to, n))
124+
return __clear_user(to, n);
125+
return n;
126+
}
82127
#endif /* _ASM_X86_UACCESS_64_H */

‎arch/x86/lib/clear_page_64.S

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/* SPDX-License-Identifier: GPL-2.0-only */
22
#include <linux/linkage.h>
3+
#include <asm/asm.h>
34
#include <asm/export.h>
45

56
/*
@@ -50,3 +51,140 @@ SYM_FUNC_START(clear_page_erms)
5051
RET
5152
SYM_FUNC_END(clear_page_erms)
5253
EXPORT_SYMBOL_GPL(clear_page_erms)
54+
55+
/*
56+
* Default clear user-space.
57+
* Input:
58+
* rdi destination
59+
* rcx count
60+
*
61+
* Output:
62+
* rcx: uncleared bytes or 0 if successful.
63+
*/
64+
SYM_FUNC_START(clear_user_original)
65+
/*
66+
* Copy only the lower 32 bits of size as that is enough to handle the rest bytes,
67+
* i.e., no need for a 'q' suffix and thus a REX prefix.
68+
*/
69+
mov %ecx,%eax
70+
shr $3,%rcx
71+
jz .Lrest_bytes
72+
73+
# do the qwords first
74+
.p2align 4
75+
.Lqwords:
76+
movq $0,(%rdi)
77+
lea 8(%rdi),%rdi
78+
dec %rcx
79+
jnz .Lqwords
80+
81+
.Lrest_bytes:
82+
and $7, %eax
83+
jz .Lexit
84+
85+
# now do the rest bytes
86+
.Lbytes:
87+
movb $0,(%rdi)
88+
inc %rdi
89+
dec %eax
90+
jnz .Lbytes
91+
92+
.Lexit:
93+
/*
94+
* %rax still needs to be cleared in the exception case because this function is called
95+
* from inline asm and the compiler expects %rax to be zero when exiting the inline asm,
96+
* in case it might reuse it somewhere.
97+
*/
98+
xor %eax,%eax
99+
RET
100+
101+
.Lqwords_exception:
102+
# convert remaining qwords back into bytes to return to caller
103+
shl $3, %rcx
104+
and $7, %eax
105+
add %rax,%rcx
106+
jmp .Lexit
107+
108+
.Lbytes_exception:
109+
mov %eax,%ecx
110+
jmp .Lexit
111+
112+
_ASM_EXTABLE_UA(.Lqwords, .Lqwords_exception)
113+
_ASM_EXTABLE_UA(.Lbytes, .Lbytes_exception)
114+
SYM_FUNC_END(clear_user_original)
115+
EXPORT_SYMBOL(clear_user_original)
116+
117+
/*
118+
* Alternative clear user-space when CPU feature X86_FEATURE_REP_GOOD is
119+
* present.
120+
* Input:
121+
* rdi destination
122+
* rcx count
123+
*
124+
* Output:
125+
* rcx: uncleared bytes or 0 if successful.
126+
*/
127+
SYM_FUNC_START(clear_user_rep_good)
128+
# call the original thing for less than a cacheline
129+
cmp $64, %rcx
130+
jb clear_user_original
131+
132+
.Lprep:
133+
# copy lower 32-bits for rest bytes
134+
mov %ecx, %edx
135+
shr $3, %rcx
136+
jz .Lrep_good_rest_bytes
137+
138+
.Lrep_good_qwords:
139+
rep stosq
140+
141+
.Lrep_good_rest_bytes:
142+
and $7, %edx
143+
jz .Lrep_good_exit
144+
145+
.Lrep_good_bytes:
146+
mov %edx, %ecx
147+
rep stosb
148+
149+
.Lrep_good_exit:
150+
# see .Lexit comment above
151+
xor %eax, %eax
152+
RET
153+
154+
.Lrep_good_qwords_exception:
155+
# convert remaining qwords back into bytes to return to caller
156+
shl $3, %rcx
157+
and $7, %edx
158+
add %rdx, %rcx
159+
jmp .Lrep_good_exit
160+
161+
_ASM_EXTABLE_UA(.Lrep_good_qwords, .Lrep_good_qwords_exception)
162+
_ASM_EXTABLE_UA(.Lrep_good_bytes, .Lrep_good_exit)
163+
SYM_FUNC_END(clear_user_rep_good)
164+
EXPORT_SYMBOL(clear_user_rep_good)
165+
166+
/*
167+
* Alternative clear user-space when CPU feature X86_FEATURE_ERMS is present.
168+
* Input:
169+
* rdi destination
170+
* rcx count
171+
*
172+
* Output:
173+
* rcx: uncleared bytes or 0 if successful.
174+
*
175+
*/
176+
SYM_FUNC_START(clear_user_erms)
177+
# call the original thing for less than a cacheline
178+
cmp $64, %rcx
179+
jb clear_user_original
180+
181+
.Lerms_bytes:
182+
rep stosb
183+
184+
.Lerms_exit:
185+
xorl %eax,%eax
186+
RET
187+
188+
_ASM_EXTABLE_UA(.Lerms_bytes, .Lerms_exit)
189+
SYM_FUNC_END(clear_user_erms)
190+
EXPORT_SYMBOL(clear_user_erms)

‎arch/x86/lib/usercopy_64.c

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -14,46 +14,6 @@
1414
* Zero Userspace
1515
*/
1616

17-
unsigned long __clear_user(void __user *addr, unsigned long size)
18-
{
19-
long __d0;
20-
might_fault();
21-
/* no memory constraint because it doesn't change any memory gcc knows
22-
about */
23-
stac();
24-
asm volatile(
25-
" testq %[size8],%[size8]\n"
26-
" jz 4f\n"
27-
" .align 16\n"
28-
"0: movq $0,(%[dst])\n"
29-
" addq $8,%[dst]\n"
30-
" decl %%ecx ; jnz 0b\n"
31-
"4: movq %[size1],%%rcx\n"
32-
" testl %%ecx,%%ecx\n"
33-
" jz 2f\n"
34-
"1: movb $0,(%[dst])\n"
35-
" incq %[dst]\n"
36-
" decl %%ecx ; jnz 1b\n"
37-
"2:\n"
38-
39-
_ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN8, %[size1])
40-
_ASM_EXTABLE_UA(1b, 2b)
41-
42-
: [size8] "=&c"(size), [dst] "=&D" (__d0)
43-
: [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));
44-
clac();
45-
return size;
46-
}
47-
EXPORT_SYMBOL(__clear_user);
48-
49-
unsigned long clear_user(void __user *to, unsigned long n)
50-
{
51-
if (access_ok(to, n))
52-
return __clear_user(to, n);
53-
return n;
54-
}
55-
EXPORT_SYMBOL(clear_user);
56-
5717
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
5818
/**
5919
* clean_cache_range - write back a cache range with CLWB

‎arch/x86/mm/fault.c

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -769,6 +769,8 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code,
769769
unsigned long address, struct task_struct *tsk)
770770
{
771771
const char *loglvl = task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG;
772+
/* This is a racy snapshot, but it's better than nothing. */
773+
int cpu = raw_smp_processor_id();
772774

773775
if (!unhandled_signal(tsk, SIGSEGV))
774776
return;
@@ -782,6 +784,14 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code,
782784

783785
print_vma_addr(KERN_CONT " in ", regs->ip);
784786

787+
/*
788+
* Dump the likely CPU where the fatal segfault happened.
789+
* This can help identify faulty hardware.
790+
*/
791+
printk(KERN_CONT " likely on CPU %d (core %d, socket %d)", cpu,
792+
topology_core_id(cpu), topology_physical_package_id(cpu));
793+
794+
785795
printk(KERN_CONT "\n");
786796

787797
show_opcodes(regs, loglvl);

‎tools/objtool/check.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1073,6 +1073,9 @@ static const char *uaccess_safe_builtin[] = {
10731073
"copy_mc_fragile_handle_tail",
10741074
"copy_mc_enhanced_fast_string",
10751075
"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1076+
"clear_user_erms",
1077+
"clear_user_rep_good",
1078+
"clear_user_original",
10761079
NULL
10771080
};
10781081

0 commit comments

Comments
 (0)