Skip to content

Commit 655ee55

Browse files
willdeaconwilldeacon
committed
Merge branch 'for-next/sve' into for-next/core
* for-next/sve: arm64/sve: Fix warnings when SVE is disabled arm64/sve: Add stub for sve_max_virtualisable_vl() arm64/sve: Track vector lengths for tasks in an array arm64/sve: Explicitly load vector length when restoring SVE state arm64/sve: Put system wide vector length information into structs arm64/sve: Use accessor functions for vector lengths in thread_struct arm64/sve: Rename find_supported_vector_length() arm64/sve: Make access to FFR optional arm64/sve: Make sve_state_size() static arm64/sve: Remove sve_load_from_fpsimd_state() arm64/fp: Reindent fpsimd_save()
2 parents 3d9c831 + 04ee53a commit 655ee55

File tree

11 files changed

+386
-212
lines changed

11 files changed

+386
-212
lines changed

‎arch/arm64/include/asm/fpsimd.h

Lines changed: 95 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -62,15 +62,13 @@ static inline size_t sve_ffr_offset(int vl)
6262

6363
static inline void *sve_pffr(struct thread_struct *thread)
6464
{
65-
return (char *)thread->sve_state + sve_ffr_offset(thread->sve_vl);
65+
return (char *)thread->sve_state + sve_ffr_offset(thread_get_sve_vl(thread));
6666
}
6767

68-
extern void sve_save_state(void *state, u32 *pfpsr);
68+
extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
6969
extern void sve_load_state(void const *state, u32 const *pfpsr,
70-
unsigned long vq_minus_1);
71-
extern void sve_flush_live(unsigned long vq_minus_1);
72-
extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state,
73-
unsigned long vq_minus_1);
70+
int restore_ffr);
71+
extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
7472
extern unsigned int sve_get_vl(void);
7573
extern void sve_set_vq(unsigned long vq_minus_1);
7674

@@ -79,10 +77,6 @@ extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);
7977

8078
extern u64 read_zcr_features(void);
8179

82-
extern int __ro_after_init sve_max_vl;
83-
extern int __ro_after_init sve_max_virtualisable_vl;
84-
extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
85-
8680
/*
8781
* Helpers to translate bit indices in sve_vq_map to VQ values (and
8882
* vice versa). This allows find_next_bit() to be used to find the
@@ -98,15 +92,29 @@ static inline unsigned int __bit_to_vq(unsigned int bit)
9892
return SVE_VQ_MAX - bit;
9993
}
10094

101-
/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
102-
static inline bool sve_vq_available(unsigned int vq)
103-
{
104-
return test_bit(__vq_to_bit(vq), sve_vq_map);
105-
}
10695

107-
#ifdef CONFIG_ARM64_SVE
96+
struct vl_info {
97+
enum vec_type type;
98+
const char *name; /* For display purposes */
10899

109-
extern size_t sve_state_size(struct task_struct const *task);
100+
/* Minimum supported vector length across all CPUs */
101+
int min_vl;
102+
103+
/* Maximum supported vector length across all CPUs */
104+
int max_vl;
105+
int max_virtualisable_vl;
106+
107+
/*
108+
* Set of available vector lengths,
109+
* where length vq encoded as bit __vq_to_bit(vq):
110+
*/
111+
DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
112+
113+
/* Set of vector lengths present on at least one cpu: */
114+
DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
115+
};
116+
117+
#ifdef CONFIG_ARM64_SVE
110118

111119
extern void sve_alloc(struct task_struct *task);
112120
extern void fpsimd_release_task(struct task_struct *task);
@@ -143,18 +151,75 @@ static inline void sve_user_enable(void)
143151
* Probing and setup functions.
144152
* Calls to these functions must be serialised with one another.
145153
*/
146-
extern void __init sve_init_vq_map(void);
147-
extern void sve_update_vq_map(void);
148-
extern int sve_verify_vq_map(void);
154+
enum vec_type;
155+
156+
extern void __init vec_init_vq_map(enum vec_type type);
157+
extern void vec_update_vq_map(enum vec_type type);
158+
extern int vec_verify_vq_map(enum vec_type type);
149159
extern void __init sve_setup(void);
150160

161+
extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
162+
163+
static inline void write_vl(enum vec_type type, u64 val)
164+
{
165+
u64 tmp;
166+
167+
switch (type) {
168+
#ifdef CONFIG_ARM64_SVE
169+
case ARM64_VEC_SVE:
170+
tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
171+
write_sysreg_s(tmp | val, SYS_ZCR_EL1);
172+
break;
173+
#endif
174+
default:
175+
WARN_ON_ONCE(1);
176+
break;
177+
}
178+
}
179+
180+
static inline int vec_max_vl(enum vec_type type)
181+
{
182+
return vl_info[type].max_vl;
183+
}
184+
185+
static inline int vec_max_virtualisable_vl(enum vec_type type)
186+
{
187+
return vl_info[type].max_virtualisable_vl;
188+
}
189+
190+
static inline int sve_max_vl(void)
191+
{
192+
return vec_max_vl(ARM64_VEC_SVE);
193+
}
194+
195+
static inline int sve_max_virtualisable_vl(void)
196+
{
197+
return vec_max_virtualisable_vl(ARM64_VEC_SVE);
198+
}
199+
200+
/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
201+
static inline bool vq_available(enum vec_type type, unsigned int vq)
202+
{
203+
return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
204+
}
205+
206+
static inline bool sve_vq_available(unsigned int vq)
207+
{
208+
return vq_available(ARM64_VEC_SVE, vq);
209+
}
210+
151211
#else /* ! CONFIG_ARM64_SVE */
152212

153213
static inline void sve_alloc(struct task_struct *task) { }
154214
static inline void fpsimd_release_task(struct task_struct *task) { }
155215
static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
156216
static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
157217

218+
static inline int sve_max_virtualisable_vl(void)
219+
{
220+
return 0;
221+
}
222+
158223
static inline int sve_set_current_vl(unsigned long arg)
159224
{
160225
return -EINVAL;
@@ -165,14 +230,21 @@ static inline int sve_get_current_vl(void)
165230
return -EINVAL;
166231
}
167232

233+
static inline int sve_max_vl(void)
234+
{
235+
return -EINVAL;
236+
}
237+
238+
static inline bool sve_vq_available(unsigned int vq) { return false; }
239+
168240
static inline void sve_user_disable(void) { BUILD_BUG(); }
169241
static inline void sve_user_enable(void) { BUILD_BUG(); }
170242

171243
#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
172244

173-
static inline void sve_init_vq_map(void) { }
174-
static inline void sve_update_vq_map(void) { }
175-
static inline int sve_verify_vq_map(void) { return 0; }
245+
static inline void vec_init_vq_map(enum vec_type t) { }
246+
static inline void vec_update_vq_map(enum vec_type t) { }
247+
static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
176248
static inline void sve_setup(void) { }
177249

178250
#endif /* ! CONFIG_ARM64_SVE */

‎arch/arm64/include/asm/fpsimdmacros.h

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -217,37 +217,40 @@
217217
.macro sve_flush_z
218218
_for n, 0, 31, _sve_flush_z \n
219219
.endm
220-
.macro sve_flush_p_ffr
220+
.macro sve_flush_p
221221
_for n, 0, 15, _sve_pfalse \n
222+
.endm
223+
.macro sve_flush_ffr
222224
_sve_wrffr 0
223225
.endm
224226

225-
.macro sve_save nxbase, xpfpsr, nxtmp
227+
.macro sve_save nxbase, xpfpsr, save_ffr, nxtmp
226228
_for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
227229
_for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
230+
cbz \save_ffr, 921f
228231
_sve_rdffr 0
229232
_sve_str_p 0, \nxbase
230233
_sve_ldr_p 0, \nxbase, -16
231-
234+
b 922f
235+
921:
236+
str xzr, [x\nxbase] // Zero out FFR
237+
922:
232238
mrs x\nxtmp, fpsr
233239
str w\nxtmp, [\xpfpsr]
234240
mrs x\nxtmp, fpcr
235241
str w\nxtmp, [\xpfpsr, #4]
236242
.endm
237243

238-
.macro __sve_load nxbase, xpfpsr, nxtmp
244+
.macro sve_load nxbase, xpfpsr, restore_ffr, nxtmp
239245
_for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
246+
cbz \restore_ffr, 921f
240247
_sve_ldr_p 0, \nxbase
241248
_sve_wrffr 0
249+
921:
242250
_for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16
243251

244252
ldr w\nxtmp, [\xpfpsr]
245253
msr fpsr, x\nxtmp
246254
ldr w\nxtmp, [\xpfpsr, #4]
247255
msr fpcr, x\nxtmp
248256
.endm
249-
250-
.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
251-
sve_load_vq \xvqminus1, x\nxtmp, \xtmp2
252-
__sve_load \nxbase, \xpfpsr, \nxtmp
253-
.endm

‎arch/arm64/include/asm/processor.h

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,11 @@ struct debug_info {
115115
#endif
116116
};
117117

118+
enum vec_type {
119+
ARM64_VEC_SVE = 0,
120+
ARM64_VEC_MAX,
121+
};
122+
118123
struct cpu_context {
119124
unsigned long x19;
120125
unsigned long x20;
@@ -147,8 +152,8 @@ struct thread_struct {
147152

148153
unsigned int fpsimd_cpu;
149154
void *sve_state; /* SVE registers, if any */
150-
unsigned int sve_vl; /* SVE vector length */
151-
unsigned int sve_vl_onexec; /* SVE vl after next exec */
155+
unsigned int vl[ARM64_VEC_MAX]; /* vector length */
156+
unsigned int vl_onexec[ARM64_VEC_MAX]; /* vl after next exec */
152157
unsigned long fault_address; /* fault info */
153158
unsigned long fault_code; /* ESR_EL1 value */
154159
struct debug_info debug; /* debugging */
@@ -164,6 +169,46 @@ struct thread_struct {
164169
u64 sctlr_user;
165170
};
166171

172+
static inline unsigned int thread_get_vl(struct thread_struct *thread,
173+
enum vec_type type)
174+
{
175+
return thread->vl[type];
176+
}
177+
178+
static inline unsigned int thread_get_sve_vl(struct thread_struct *thread)
179+
{
180+
return thread_get_vl(thread, ARM64_VEC_SVE);
181+
}
182+
183+
unsigned int task_get_vl(const struct task_struct *task, enum vec_type type);
184+
void task_set_vl(struct task_struct *task, enum vec_type type,
185+
unsigned long vl);
186+
void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
187+
unsigned long vl);
188+
unsigned int task_get_vl_onexec(const struct task_struct *task,
189+
enum vec_type type);
190+
191+
static inline unsigned int task_get_sve_vl(const struct task_struct *task)
192+
{
193+
return task_get_vl(task, ARM64_VEC_SVE);
194+
}
195+
196+
static inline void task_set_sve_vl(struct task_struct *task, unsigned long vl)
197+
{
198+
task_set_vl(task, ARM64_VEC_SVE, vl);
199+
}
200+
201+
static inline unsigned int task_get_sve_vl_onexec(const struct task_struct *task)
202+
{
203+
return task_get_vl_onexec(task, ARM64_VEC_SVE);
204+
}
205+
206+
static inline void task_set_sve_vl_onexec(struct task_struct *task,
207+
unsigned long vl)
208+
{
209+
task_set_vl_onexec(task, ARM64_VEC_SVE, vl);
210+
}
211+
167212
#define SCTLR_USER_MASK \
168213
(SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB | \
169214
SCTLR_EL1_TCF0_MASK)

‎arch/arm64/include/asm/thread_info.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ int arch_dup_task_struct(struct task_struct *dst,
7878
#define TIF_SINGLESTEP 21
7979
#define TIF_32BIT 22 /* 32bit process */
8080
#define TIF_SVE 23 /* Scalable Vector Extension in use */
81-
#define TIF_SVE_VL_INHERIT 24 /* Inherit sve_vl_onexec across exec */
81+
#define TIF_SVE_VL_INHERIT 24 /* Inherit SVE vl_onexec across exec */
8282
#define TIF_SSBD 25 /* Wants SSB mitigation */
8383
#define TIF_TAGGED_ADDR 26 /* Allow tagged user addresses */
8484

‎arch/arm64/kernel/cpufeature.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -941,7 +941,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
941941

942942
if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
943943
init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
944-
sve_init_vq_map();
944+
vec_init_vq_map(ARM64_VEC_SVE);
945945
}
946946

947947
if (id_aa64pfr1_mte(info->reg_id_aa64pfr1))
@@ -1175,7 +1175,7 @@ void update_cpu_features(int cpu,
11751175
/* Probe vector lengths, unless we already gave up on SVE */
11761176
if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
11771177
!system_capabilities_finalized())
1178-
sve_update_vq_map();
1178+
vec_update_vq_map(ARM64_VEC_SVE);
11791179
}
11801180

11811181
/*
@@ -2760,7 +2760,7 @@ static void verify_sve_features(void)
27602760
unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;
27612761
unsigned int len = zcr & ZCR_ELx_LEN_MASK;
27622762

2763-
if (len < safe_len || sve_verify_vq_map()) {
2763+
if (len < safe_len || vec_verify_vq_map(ARM64_VEC_SVE)) {
27642764
pr_crit("CPU%d: SVE: vector length support mismatch\n",
27652765
smp_processor_id());
27662766
cpu_die_early();

‎arch/arm64/kernel/entry-fpsimd.S

Lines changed: 11 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,10 @@ SYM_FUNC_END(fpsimd_load_state)
3838
*
3939
* x0 - pointer to buffer for state
4040
* x1 - pointer to storage for FPSR
41+
* x2 - Save FFR if non-zero
4142
*/
4243
SYM_FUNC_START(sve_save_state)
43-
sve_save 0, x1, 2
44+
sve_save 0, x1, x2, 3
4445
ret
4546
SYM_FUNC_END(sve_save_state)
4647

@@ -49,10 +50,10 @@ SYM_FUNC_END(sve_save_state)
4950
*
5051
* x0 - pointer to buffer for state
5152
* x1 - pointer to storage for FPSR
52-
* x2 - VQ-1
53+
* x2 - Restore FFR if non-zero
5354
*/
5455
SYM_FUNC_START(sve_load_state)
55-
sve_load 0, x1, x2, 3, x4
56+
sve_load 0, x1, x2, 4
5657
ret
5758
SYM_FUNC_END(sve_load_state)
5859

@@ -66,35 +67,22 @@ SYM_FUNC_START(sve_set_vq)
6667
ret
6768
SYM_FUNC_END(sve_set_vq)
6869

69-
/*
70-
* Load SVE state from FPSIMD state.
71-
*
72-
* x0 = pointer to struct fpsimd_state
73-
* x1 = VQ - 1
74-
*
75-
* Each SVE vector will be loaded with the first 128-bits taken from FPSIMD
76-
* and the rest zeroed. All the other SVE registers will be zeroed.
77-
*/
78-
SYM_FUNC_START(sve_load_from_fpsimd_state)
79-
sve_load_vq x1, x2, x3
80-
fpsimd_restore x0, 8
81-
sve_flush_p_ffr
82-
ret
83-
SYM_FUNC_END(sve_load_from_fpsimd_state)
84-
8570
/*
8671
* Zero all SVE registers but the first 128-bits of each vector
8772
*
8873
* VQ must already be configured by caller, any further updates of VQ
8974
* will need to ensure that the register state remains valid.
9075
*
91-
* x0 = VQ - 1
76+
* x0 = include FFR?
77+
* x1 = VQ - 1
9278
*/
9379
SYM_FUNC_START(sve_flush_live)
94-
cbz x0, 1f // A VQ-1 of 0 is 128 bits so no extra Z state
80+
cbz x1, 1f // A VQ-1 of 0 is 128 bits so no extra Z state
9581
sve_flush_z
96-
1: sve_flush_p_ffr
97-
ret
82+
1: sve_flush_p
83+
tbz x0, #0, 2f
84+
sve_flush_ffr
85+
2: ret
9886
SYM_FUNC_END(sve_flush_live)
9987

10088
#endif /* CONFIG_ARM64_SVE */

0 commit comments

Comments
 (0)