Skip to content

Commit 7066248

Browse files
willdeaconwilldeacon
committed
Merge branch 'for-next/mte' into for-next/core
* for-next/mte: kasan: Extend KASAN mode kernel parameter arm64: mte: Add asymmetric mode support arm64: mte: CPU feature detection for Asymm MTE arm64: mte: Bitfield definitions for Asymm MTE kasan: Remove duplicate of kasan_flag_async arm64: kasan: mte: move GCR_EL1 switch to task switch when KASAN disabled
2 parents dc6bab1 + 2d27e58 commit 7066248

File tree

15 files changed

+151
-39
lines changed

15 files changed

+151
-39
lines changed

‎Documentation/dev-tools/kasan.rst

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,14 +194,17 @@ additional boot parameters that allow disabling KASAN or controlling features:
194194

195195
- ``kasan=off`` or ``=on`` controls whether KASAN is enabled (default: ``on``).
196196

197-
- ``kasan.mode=sync`` or ``=async`` controls whether KASAN is configured in
198-
synchronous or asynchronous mode of execution (default: ``sync``).
197+
- ``kasan.mode=sync``, ``=async`` or ``=asymm`` controls whether KASAN
198+
is configured in synchronous, asynchronous or asymmetric mode of
199+
execution (default: ``sync``).
199200
Synchronous mode: a bad access is detected immediately when a tag
200201
check fault occurs.
201202
Asynchronous mode: a bad access detection is delayed. When a tag check
202203
fault occurs, the information is stored in hardware (in the TFSR_EL1
203204
register for arm64). The kernel periodically checks the hardware and
204205
only reports tag faults during these checks.
206+
Asymmetric mode: a bad access is detected synchronously on reads and
207+
asynchronously on writes.
205208

206209
- ``kasan.stacktrace=off`` or ``=on`` disables or enables alloc and free stack
207210
traces collection (default: ``on``).

‎arch/arm64/include/asm/memory.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ static inline const void *__tag_set(const void *addr, u8 tag)
243243
#ifdef CONFIG_KASAN_HW_TAGS
244244
#define arch_enable_tagging_sync() mte_enable_kernel_sync()
245245
#define arch_enable_tagging_async() mte_enable_kernel_async()
246+
#define arch_enable_tagging_asymm() mte_enable_kernel_asymm()
246247
#define arch_force_async_tag_fault() mte_check_tfsr_exit()
247248
#define arch_get_random_tag() mte_get_random_tag()
248249
#define arch_get_mem_tag(addr) mte_get_mem_tag(addr)

‎arch/arm64/include/asm/mte-kasan.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
130130

131131
void mte_enable_kernel_sync(void);
132132
void mte_enable_kernel_async(void);
133+
void mte_enable_kernel_asymm(void);
133134

134135
#else /* CONFIG_ARM64_MTE */
135136

@@ -161,6 +162,10 @@ static inline void mte_enable_kernel_async(void)
161162
{
162163
}
163164

165+
static inline void mte_enable_kernel_asymm(void)
166+
{
167+
}
168+
164169
#endif /* CONFIG_ARM64_MTE */
165170

166171
#endif /* __ASSEMBLY__ */

‎arch/arm64/include/asm/mte.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,11 +88,11 @@ static inline int mte_ptrace_copy_tags(struct task_struct *child,
8888

8989
#ifdef CONFIG_KASAN_HW_TAGS
9090
/* Whether the MTE asynchronous mode is enabled. */
91-
DECLARE_STATIC_KEY_FALSE(mte_async_mode);
91+
DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);
9292

93-
static inline bool system_uses_mte_async_mode(void)
93+
static inline bool system_uses_mte_async_or_asymm_mode(void)
9494
{
95-
return static_branch_unlikely(&mte_async_mode);
95+
return static_branch_unlikely(&mte_async_or_asymm_mode);
9696
}
9797

9898
void mte_check_tfsr_el1(void);
@@ -121,7 +121,7 @@ static inline void mte_check_tfsr_exit(void)
121121
mte_check_tfsr_el1();
122122
}
123123
#else
124-
static inline bool system_uses_mte_async_mode(void)
124+
static inline bool system_uses_mte_async_or_asymm_mode(void)
125125
{
126126
return false;
127127
}

‎arch/arm64/include/asm/sysreg.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,7 @@
626626
#define SCTLR_ELx_TCF_NONE (UL(0x0) << SCTLR_ELx_TCF_SHIFT)
627627
#define SCTLR_ELx_TCF_SYNC (UL(0x1) << SCTLR_ELx_TCF_SHIFT)
628628
#define SCTLR_ELx_TCF_ASYNC (UL(0x2) << SCTLR_ELx_TCF_SHIFT)
629+
#define SCTLR_ELx_TCF_ASYMM (UL(0x3) << SCTLR_ELx_TCF_SHIFT)
629630
#define SCTLR_ELx_TCF_MASK (UL(0x3) << SCTLR_ELx_TCF_SHIFT)
630631

631632
#define SCTLR_ELx_ENIA_SHIFT 31
@@ -671,6 +672,7 @@
671672
#define SCTLR_EL1_TCF0_NONE (UL(0x0) << SCTLR_EL1_TCF0_SHIFT)
672673
#define SCTLR_EL1_TCF0_SYNC (UL(0x1) << SCTLR_EL1_TCF0_SHIFT)
673674
#define SCTLR_EL1_TCF0_ASYNC (UL(0x2) << SCTLR_EL1_TCF0_SHIFT)
675+
#define SCTLR_EL1_TCF0_ASYMM (UL(0x3) << SCTLR_EL1_TCF0_SHIFT)
674676
#define SCTLR_EL1_TCF0_MASK (UL(0x3) << SCTLR_EL1_TCF0_SHIFT)
675677

676678
#define SCTLR_EL1_BT1 (BIT(36))
@@ -812,6 +814,7 @@
812814
#define ID_AA64PFR1_MTE_NI 0x0
813815
#define ID_AA64PFR1_MTE_EL0 0x1
814816
#define ID_AA64PFR1_MTE 0x2
817+
#define ID_AA64PFR1_MTE_ASYMM 0x3
815818

816819
/* id_aa64zfr0 */
817820
#define ID_AA64ZFR0_F64MM_SHIFT 56

‎arch/arm64/include/asm/uaccess.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,13 +191,13 @@ static inline void __uaccess_enable_tco(void)
191191
*/
192192
static inline void __uaccess_disable_tco_async(void)
193193
{
194-
if (system_uses_mte_async_mode())
194+
if (system_uses_mte_async_or_asymm_mode())
195195
__uaccess_disable_tco();
196196
}
197197

198198
static inline void __uaccess_enable_tco_async(void)
199199
{
200-
if (system_uses_mte_async_mode())
200+
if (system_uses_mte_async_or_asymm_mode())
201201
__uaccess_enable_tco();
202202
}
203203

‎arch/arm64/kernel/cpufeature.c

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2331,6 +2331,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
23312331
.sign = FTR_UNSIGNED,
23322332
.cpu_enable = cpu_enable_mte,
23332333
},
2334+
{
2335+
.desc = "Asymmetric MTE Tag Check Fault",
2336+
.capability = ARM64_MTE_ASYMM,
2337+
.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2338+
.matches = has_cpuid_feature,
2339+
.sys_reg = SYS_ID_AA64PFR1_EL1,
2340+
.field_pos = ID_AA64PFR1_MTE_SHIFT,
2341+
.min_field_value = ID_AA64PFR1_MTE_ASYMM,
2342+
.sign = FTR_UNSIGNED,
2343+
},
23342344
#endif /* CONFIG_ARM64_MTE */
23352345
{
23362346
.desc = "RCpc load-acquire (LDAPR)",

‎arch/arm64/kernel/entry.S

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -168,20 +168,20 @@ alternative_else_nop_endif
168168

169169
.macro mte_set_kernel_gcr, tmp, tmp2
170170
#ifdef CONFIG_KASAN_HW_TAGS
171-
alternative_if_not ARM64_MTE
171+
alternative_cb kasan_hw_tags_enable
172172
b 1f
173-
alternative_else_nop_endif
173+
alternative_cb_end
174174
mov \tmp, KERNEL_GCR_EL1
175175
msr_s SYS_GCR_EL1, \tmp
176176
1:
177177
#endif
178178
.endm
179179

180180
.macro mte_set_user_gcr, tsk, tmp, tmp2
181-
#ifdef CONFIG_ARM64_MTE
182-
alternative_if_not ARM64_MTE
181+
#ifdef CONFIG_KASAN_HW_TAGS
182+
alternative_cb kasan_hw_tags_enable
183183
b 1f
184-
alternative_else_nop_endif
184+
alternative_cb_end
185185
ldr \tmp, [\tsk, #THREAD_MTE_CTRL]
186186

187187
mte_set_gcr \tmp, \tmp2

‎arch/arm64/kernel/mte.c

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,12 @@
2626
static DEFINE_PER_CPU_READ_MOSTLY(u64, mte_tcf_preferred);
2727

2828
#ifdef CONFIG_KASAN_HW_TAGS
29-
/* Whether the MTE asynchronous mode is enabled. */
30-
DEFINE_STATIC_KEY_FALSE(mte_async_mode);
31-
EXPORT_SYMBOL_GPL(mte_async_mode);
29+
/*
30+
* The asynchronous and asymmetric MTE modes have the same behavior for
31+
* store operations. This flag is set when either of these modes is enabled.
32+
*/
33+
DEFINE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);
34+
EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode);
3235
#endif
3336

3437
static void mte_sync_page_tags(struct page *page, pte_t old_pte,
@@ -116,7 +119,7 @@ void mte_enable_kernel_sync(void)
116119
* Make sure we enter this function when no PE has set
117120
* async mode previously.
118121
*/
119-
WARN_ONCE(system_uses_mte_async_mode(),
122+
WARN_ONCE(system_uses_mte_async_or_asymm_mode(),
120123
"MTE async mode enabled system wide!");
121124

122125
__mte_enable_kernel("synchronous", SCTLR_ELx_TCF_SYNC);
@@ -134,8 +137,34 @@ void mte_enable_kernel_async(void)
134137
* mode in between sync and async, this strategy needs
135138
* to be reviewed.
136139
*/
137-
if (!system_uses_mte_async_mode())
138-
static_branch_enable(&mte_async_mode);
140+
if (!system_uses_mte_async_or_asymm_mode())
141+
static_branch_enable(&mte_async_or_asymm_mode);
142+
}
143+
144+
void mte_enable_kernel_asymm(void)
145+
{
146+
if (cpus_have_cap(ARM64_MTE_ASYMM)) {
147+
__mte_enable_kernel("asymmetric", SCTLR_ELx_TCF_ASYMM);
148+
149+
/*
150+
* MTE asymm mode behaves as async mode for store
151+
* operations. The mode is set system wide by the
152+
* first PE that executes this function.
153+
*
154+
* Note: If in future KASAN acquires a runtime switching
155+
* mode in between sync and async, this strategy needs
156+
* to be reviewed.
157+
*/
158+
if (!system_uses_mte_async_or_asymm_mode())
159+
static_branch_enable(&mte_async_or_asymm_mode);
160+
} else {
161+
/*
162+
* If the CPU does not support MTE asymmetric mode the
163+
* kernel falls back on synchronous mode which is the
164+
* default for kasan=on.
165+
*/
166+
mte_enable_kernel_sync();
167+
}
139168
}
140169
#endif
141170

@@ -179,6 +208,30 @@ static void mte_update_sctlr_user(struct task_struct *task)
179208
task->thread.sctlr_user = sctlr;
180209
}
181210

211+
static void mte_update_gcr_excl(struct task_struct *task)
212+
{
213+
/*
214+
* SYS_GCR_EL1 will be set to current->thread.mte_ctrl value by
215+
* mte_set_user_gcr() in kernel_exit, but only if KASAN is enabled.
216+
*/
217+
if (kasan_hw_tags_enabled())
218+
return;
219+
220+
write_sysreg_s(
221+
((task->thread.mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) &
222+
SYS_GCR_EL1_EXCL_MASK) | SYS_GCR_EL1_RRND,
223+
SYS_GCR_EL1);
224+
}
225+
226+
void __init kasan_hw_tags_enable(struct alt_instr *alt, __le32 *origptr,
227+
__le32 *updptr, int nr_inst)
228+
{
229+
BUG_ON(nr_inst != 1); /* Branch -> NOP */
230+
231+
if (kasan_hw_tags_enabled())
232+
*updptr = cpu_to_le32(aarch64_insn_gen_nop());
233+
}
234+
182235
void mte_thread_init_user(void)
183236
{
184237
if (!system_supports_mte())
@@ -198,6 +251,7 @@ void mte_thread_switch(struct task_struct *next)
198251
return;
199252

200253
mte_update_sctlr_user(next);
254+
mte_update_gcr_excl(next);
201255

202256
/*
203257
* Check if an async tag exception occurred at EL1.
@@ -243,6 +297,7 @@ long set_mte_ctrl(struct task_struct *task, unsigned long arg)
243297
if (task == current) {
244298
preempt_disable();
245299
mte_update_sctlr_user(task);
300+
mte_update_gcr_excl(task);
246301
update_sctlr_el1(task->thread.sctlr_user);
247302
preempt_enable();
248303
}

‎arch/arm64/tools/cpucaps

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ HW_DBM
4040
KVM_PROTECTED_MODE
4141
MISMATCHED_CACHE_TYPE
4242
MTE
43+
MTE_ASYMM
4344
SPECTRE_V2
4445
SPECTRE_V3A
4546
SPECTRE_V4

0 commit comments

Comments
 (0)